Auto merge of #4674 - asutoshpalai:plugin, r=segiddins

[Plugin] Source plugins

Adds source plugin. This is in continuation of #4608.

17 files changed, 623 insertions, 123 deletions

diff --git a/lib/bundler.rb b/lib/bundler.rb
index a2260e01..0ba58e51 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -3,14 +3,15 @@ require "fileutils"
 require "pathname"
 require "rbconfig"
 require "thread"
+require "bundler/errors"
 require "bundler/environment_preserver"
 require "bundler/gem_remote_fetcher"
+require "bundler/plugin"
 require "bundler/rubygems_ext"
 require "bundler/rubygems_integration"
 require "bundler/version"
 require "bundler/constants"
 require "bundler/current_ruby"
-require "bundler/errors"
 
 module Bundler
   environment_preserver = EnvironmentPreserver.new(ENV, %w(PATH GEM_PATH))
@@ -38,7 +39,6 @@ module Bundler
   autoload :MatchPlatform,          "bundler/match_platform"
   autoload :Mirror,                 "bundler/mirror"
   autoload :Mirrors,                "bundler/mirror"
-  autoload :Plugin,                 "bundler/plugin"
   autoload :RemoteSpecification,    "bundler/remote_specification"
   autoload :Resolver,               "bundler/resolver"
   autoload :Retry,                  "bundler/retry"
diff --git a/lib/bundler/cli/update.rb b/lib/bundler/cli/update.rb
index 33b0557f..bef62f3b 100644
--- a/lib/bundler/cli/update.rb
+++ b/lib/bundler/cli/update.rb
@@ -10,6 +10,8 @@ module Bundler
     def run
       Bundler.ui.level = "error" if options[:quiet]
 
+      Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.settings[:plugins]
+
       sources = Array(options[:source])
       groups  = Array(options[:group]).map(&:to_sym)
 
diff --git a/lib/bundler/dsl.rb b/lib/bundler/dsl.rb
index 358784a9..04ef641b 100644
--- a/lib/bundler/dsl.rb
+++ b/lib/bundler/dsl.rb
@@ -125,11 +125,26 @@ module Bundler
       @dependencies << dep
     end
 
-    def source(source, &blk)
-      source = normalize_source(source)
-      if block_given?
+    def source(source, *args, &blk)
+      options = args.last.is_a?(Hash) ? args.pop.dup : {}
+      options = normalize_hash(options)
+      if options.key?("type")
+        options["type"] = options["type"].to_s
+        unless Plugin.source?(options["type"])
+          raise "No sources available for #{options["type"]}"
+        end
+
+        unless block_given?
+          raise InvalidOption, "You need to pass a block to #source with :type option"
+        end
+
+        source_opts = options.merge("uri" => source)
+        with_source(@sources.add_plugin_source(options["type"], source_opts), &blk)
+      elsif block_given?
+        source = normalize_source(source)
         with_source(@sources.add_rubygems_source("remotes" => source), &blk)
       else
+        source = normalize_source(source)
         check_primary_source_safety(@sources)
         @sources.add_rubygems_remote(source)
       end
diff --git a/lib/bundler/lockfile_parser.rb b/lib/bundler/lockfile_parser.rb
index a57dbcac..d5b28440 100644
--- a/lib/bundler/lockfile_parser.rb
+++ b/lib/bundler/lockfile_parser.rb
@@ -22,9 +22,10 @@ module Bundler
     GIT          = "GIT".freeze
     GEM          = "GEM".freeze
     PATH         = "PATH".freeze
+    PLUGIN       = "PLUGIN SOURCE".freeze
     SPECS        = "  specs:".freeze
     OPTIONS      = /^  ([a-z]+): (.*)$/i
-    SOURCE       = [GIT, GEM, PATH].freeze
+    SOURCE       = [GIT, GEM, PATH, PLUGIN].freeze
 
     SECTIONS_BY_VERSION_INTRODUCED = {
       # The strings have to be dup'ed for old RG on Ruby 2.3+
@@ -32,6 +33,7 @@ module Bundler
       Gem::Version.create("1.0".dup) => [DEPENDENCIES, PLATFORMS, GIT, GEM, PATH].freeze,
       Gem::Version.create("1.10".dup) => [BUNDLED].freeze,
       Gem::Version.create("1.12".dup) => [RUBY].freeze,
+      Gem::Version.create("1.13".dup) => [PLUGIN].freeze,
     }.freeze
 
     KNOWN_SECTIONS = SECTIONS_BY_VERSION_INTRODUCED.values.flatten.freeze
@@ -118,17 +120,14 @@ module Bundler
   private
 
     TYPES = {
-      GIT  => Bundler::Source::Git,
-      GEM  => Bundler::Source::Rubygems,
-      PATH => Bundler::Source::Path,
+      GIT    => Bundler::Source::Git,
+      GEM    => Bundler::Source::Rubygems,
+      PATH   => Bundler::Source::Path,
+      PLUGIN => Bundler::Plugin,
     }.freeze
 
     def parse_source(line)
       case line
-      when GIT, GEM, PATH
-        @current_source = nil
-        @opts = {}
-        @type = line
       when SPECS
         case @type
         when PATH
@@ -147,6 +146,9 @@ module Bundler
             @rubygems_aggregate.add_remote(url)
           end
           @current_source = @rubygems_aggregate
+        when PLUGIN
+          @current_source = Plugin.source_from_lock(@opts)
+          @sources << @current_source
         end
       when OPTIONS
         value = $2
@@ -161,6 +163,10 @@ module Bundler
         else
           @opts[key] = value
         end
+      when *SOURCE
+        @current_source = nil
+        @opts = {}
+        @type = line
       else
         parse_spec(line)
       end
diff --git a/lib/bundler/plugin.rb b/lib/bundler/plugin.rb
index 9aabd73a..f5366d2a 100644
--- a/lib/bundler/plugin.rb
+++ b/lib/bundler/plugin.rb
@@ -10,45 +10,52 @@ module Bundler
 
     class MalformattedPlugin < PluginError; end
     class UndefinedCommandError < PluginError; end
+    class UnknownSourceError < PluginError; end
 
     PLUGIN_FILE_NAME = "plugins.rb".freeze
 
   module_function
 
     @commands = {}
+    @sources = {}
 
     # Installs a new plugin by the given name
     #
     # @param [Array<String>] names the name of plugin to be installed
-    # @param [Hash] options various parameters as described in description
-    # @option options [String] :source rubygems source to fetch the plugin gem from
-    # @option options [String] :version (optional) the version of the plugin to install
+    # @param [Hash] options various parameters as described in description.
+    #               Refer to cli/plugin for available options
     def install(names, options)
-      paths = Installer.new.install(names, options)
+      specs = Installer.new.install(names, options)
 
-      save_plugins paths
+      save_plugins names, specs
     rescue PluginError => e
-      paths.values.map {|path| Bundler.rm_rf(path) } if paths
-      Bundler.ui.error "Failed to install plugin #{name}: #{e.message}\n  #{e.backtrace.join("\n  ")}"
+      specs.values.map {|spec| Bundler.rm_rf(spec.full_gem_path) } if specs
+      Bundler.ui.error "Failed to install plugin #{name}: #{e.message}\n  #{e.backtrace[0]}"
     end
 
     # Evaluates the Gemfile with a limited DSL and installs the plugins
     # specified by plugin method
     #
     # @param [Pathname] gemfile path
+    # @param [Proc] block that can be evaluated for (inline) Gemfile
     def gemfile_install(gemfile = nil, &inline)
+      builder = DSL.new
       if block_given?
-        builder = DSL.new
         builder.instance_eval(&inline)
-        definition = builder.to_definition(nil, true)
       else
-        definition = DSL.evaluate(gemfile, nil, {})
+        builder.eval_gemfile(gemfile)
       end
-      return unless definition.dependencies.any?
+      definition = builder.to_definition(nil, true)
 
-      plugins = Installer.new.install_definition(definition)
+      return if definition.dependencies.empty?
 
-      save_plugins plugins
+      plugins = definition.dependencies.map(&:name).reject {|p| index.installed? p }
+      installed_specs = Installer.new.install_definition(definition)
+
+      save_plugins plugins, installed_specs, builder.inferred_plugins
+    rescue => e
+      Bundler.ui.error "Failed to install plugin: #{e.message}\n  #{e.backtrace[0]}"
+      raise
     end
 
     # The index object used to store the details about the plugin
@@ -71,7 +78,7 @@ module Bundler
       @commands[command] = cls
     end
 
-    # Checks if any plugins handles the command
+    # Checks if any plugin handles the command
     def command?(command)
       !index.command_plugin(command).nil?
     end
@@ -79,13 +86,41 @@ module Bundler
     # To be called from Cli class to pass the command and argument to
     # approriate plugin class
     def exec_command(command, args)
-      raise UndefinedCommandError, "Command #{command} not found" unless command? command
+      raise UndefinedCommandError, "Command `#{command}` not found" unless command? command
 
       load_plugin index.command_plugin(command) unless @commands.key? command
 
       @commands[command].new.exec(command, args)
     end
 
+    # To be called via the API to register to handle a source plugin
+    def add_source(source, cls)
+      @sources[source] = cls
+    end
+
+    # Checks if any plugin declares the source
+    def source?(name)
+      !index.source_plugin(name.to_s).nil?
+    end
+
+    # @return [Class] that handles the source. The calss includes API::Source
+    def source(name)
+      raise UnknownSourceError, "Source #{name} not found" unless source? name
+
+      load_plugin(index.source_plugin(name)) unless @sources.key? name
+
+      @sources[name]
+    end
+
+    # @param [Hash] The options that are present in the lock file
+    # @return [API::Source] the instance of the class that handles the source
+    #                       type passed in locked_opts
+    def source_from_lock(locked_opts)
+      src = source(locked_opts["type"])
+
+      src.new(locked_opts.merge("uri" => locked_opts["remote"]))
+    end
+
     # currently only intended for specs
     #
     # @return [String, nil] installed path
@@ -95,13 +130,16 @@ module Bundler
 
     # Post installation processing and registering with index
     #
-    # @param [Hash] plugins mapped to their installtion path
-    def save_plugins(plugins)
-      plugins.each do |name, path|
-        path = Pathname.new path
-        validate_plugin! path
-        register_plugin name, path
-        Bundler.ui.info "Installed plugin #{name}"
+    # @param [Array<String>] plugins list to be installed
+    # @param [Hash] specs of plugins mapped to installation path (currently they
+    #               contain all the installed specs, including plugins)
+    # @param [Array<String>] names of inferred source plugins that can be ignored
+    def save_plugins(plugins, specs, optional_plugins = [])
+      plugins.each do |name|
+        spec = specs[name]
+        validate_plugin! Pathname.new(spec.full_gem_path)
+        installed = register_plugin name, spec, optional_plugins.include?(name)
+        Bundler.ui.info "Installed plugin #{name}" if installed
       end
     end
 
@@ -110,21 +148,31 @@ module Bundler
     # At present it only checks whether it contains plugins.rb file
     #
     # @param [Pathname] plugin_path the path plugin is installed at
-    # @raise [Error] if plugins.rb file is not found
+    # @raise [MalformattedPlugin] if plugins.rb file is not found
     def validate_plugin!(plugin_path)
       plugin_file = plugin_path.join(PLUGIN_FILE_NAME)
-      raise MalformattedPlugin, "#{PLUGIN_FILE_NAME} was not found in the plugin!" unless plugin_file.file?
+      raise MalformattedPlugin, "#{PLUGIN_FILE_NAME} was not found in the plugin." unless plugin_file.file?
     end
 
     # Runs the plugins.rb file in an isolated namespace, records the plugin
     # actions it registers for and then passes the data to index to be stored.
     #
     # @param [String] name the name of the plugin
-    # @param [Pathname] path the path where the plugin is installed at
-    def register_plugin(name, path)
+    # @param [Specification] spec of installed plugin
+    # @param [Boolean] optional_plugin, removed if there is conflict with any
+    #                     other plugin (used for default source plugins)
+    #
+    # @raise [MalformattedPlugin] if plugins.rb raises any error
+    def register_plugin(name, spec, optional_plugin = false)
       commands = @commands
+      sources = @sources
 
       @commands = {}
+      @sources = {}
+
+      load_paths = spec.load_paths
+      add_to_load_path(load_paths)
+      path = Pathname.new spec.full_gem_path
 
       begin
         load path.join(PLUGIN_FILE_NAME), true
@@ -132,9 +180,16 @@ module Bundler
         raise MalformattedPlugin, "#{e.class}: #{e.message}"
       end
 
-      index.register_plugin name, path.to_s, @commands.keys
+      if optional_plugin && @sources.keys.any? {|s| source? s }
+        Bundler.rm_rf(path)
+        false
+      else
+        index.register_plugin name, path.to_s, load_paths, @commands.keys, @sources.keys
+        true
+      end
     ensure
       @commands = commands
+      @sources = sources
     end
 
     # Executes the plugins.rb file
@@ -146,11 +201,25 @@ module Bundler
       # done to avoid conflicts
       path = index.plugin_path(name)
 
+      add_to_load_path(index.load_paths(name))
+
       load path.join(PLUGIN_FILE_NAME)
+    rescue => e
+      Bundler.ui.error "Failed loading plugin #{name}: #{e.message}"
+      raise
+    end
+
+    def add_to_load_path(load_paths)
+      if insert_index = Bundler.rubygems.load_path_insert_index
+        $LOAD_PATH.insert(insert_index, *load_paths)
+      else
+        $LOAD_PATH.unshift(*load_paths)
+      end
     end
 
     class << self
-      private :load_plugin, :register_plugin, :save_plugins, :validate_plugin!
+      private :load_plugin, :register_plugin, :save_plugins, :validate_plugin!,
+        :add_to_load_path
     end
   end
 end
diff --git a/lib/bundler/plugin/api.rb b/lib/bundler/plugin/api.rb
index 9631446d..9ff4b738 100644
--- a/lib/bundler/plugin/api.rb
+++ b/lib/bundler/plugin/api.rb
@@ -23,19 +23,31 @@ module Bundler
   # and hooks).
   module Plugin
     class API
+      autoload :Source, "bundler/plugin/api/source"
       # The plugins should declare that they handle a command through this helper.
       #
       # @param [String] command being handled by them
-      # @param [Class] (optional) class that shall handle the command. If not
+      # @param [Class] (optional) class that handles the command. If not
       #                 provided, the `self` class will be used.
       def self.command(command, cls = self)
         Plugin.add_command command, cls
       end
 
-      # The cache dir to be used by the plugins for persistance storage
+      # The plugins should declare that they provide a installation source
+      # through this helper.
+      #
+      # @param [String] the source type they provide
+      # @param [Class] (optional) class that handles the source. If not
+      #                 provided, the `self` class will be used.
+      def self.source(source, cls = self)
+        cls.send :include, Bundler::Plugin::API::Source
+        Plugin.add_source source, cls
+      end
+
+      # The cache dir to be used by the plugins for storage
       #
       # @return [Pathname] path of the cache dir
-      def cache
+      def cache_dir
         Plugin.cache.join("plugins")
       end
 
@@ -48,8 +60,16 @@ module Bundler
       end
 
       def method_missing(name, *args, &blk)
-        super unless Bundler.respond_to?(name)
-        Bundler.send(name, *args, &blk)
+        return Bundler.send(name, *args, &blk) if Bundler.respond_to?(name)
+
+        return SharedHelpers.send(name, *args, &blk) if SharedHelpers.respond_to?(name)
+
+        super
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        SharedHelpers.respond_to?(name, include_private) ||
+          Bundler.respond_to?(name, include_private) || super
       end
     end
   end
diff --git a/lib/bundler/plugin/api/source.rb b/lib/bundler/plugin/api/source.rb
new file mode 100644
index 00000000..78514563
--- /dev/null
+++ b/lib/bundler/plugin/api/source.rb
@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+require "uri"
+require "digest/sha1"
+
+module Bundler
+  module Plugin
+    class API
+      # This class provides the base to build source plugins
+      # All the method here are require to build a source plugin (except
+      # `uri_hash`, `gem_install_dir`; they are helpers).
+      #
+      # Defaults for methods, where ever possible are provided which is
+      # expected to work. But, all source plugins have to override
+      # `fetch_gemspec_files` and `install`. Defaults are also not provided for
+      # `remote!`, `cache!` and `unlock!`.
+      #
+      # The defaults shall work for most situations but nevertheless they can
+      # be (preferably should be) overridden as per the plugins' needs safely
+      # (as long as they behave as expected).
+      # On overriding `initialize` you should call super first.
+      #
+      # If required plugin should override `hash`, `==` and `eql?` methods to be
+      # able to match objects representing same sources, but may be created in
+      # different situation (like form gemfile and lockfile). The default ones
+      # checks only for class and uri, but elaborate source plugins may need
+      # more comparisons (e.g. git checking on branch or tag).
+      #
+      # @!attribute [r] uri
+      #   @return [String] the remote specified with `source` block in Gemfile
+      #
+      # @!attribute [r] options
+      #   @return [String] options passed during initialization (either from
+      #     lockfile or Gemfile)
+      #
+      # @!attribute [r] name
+      #   @return [String] name that can be used to uniquely identify a source
+      #
+      # @!attribute [rw] dependency_names
+      #   @return [Array<String>] Names of dependencies that the source should
+      #     try to resolve. It is not necessary to use this list intenally. This
+      #     is present to be compatible with `Definition` and is used by
+      #     rubygems source.
+      module Source
+        attr_reader :uri, :options, :name
+        attr_accessor :dependency_names
+
+        def initialize(opts)
+          @options = opts
+          @dependency_names = []
+          @uri = opts["uri"]
+          @type = opts["type"]
+          @name = opts["name"] || "#{@type} at #{@uri}"
+        end
+
+        # This is used by the default `spec` method to constructs the
+        # Specification objects for the gems and versions that can be installed
+        # by this source plugin.
+        #
+        # Note: If the spec method is overridden, this function is not necessary
+        #
+        # @return [Array<String>] paths of the gemspec files for gems that can
+        #                         be installed
+        def fetch_gemspec_files
+          []
+        end
+
+        # Options to be saved in the lockfile so that the source plugin is able
+        # to check out same version of gem later.
+        #
+        # There options are passed when the source plugin is created from the
+        # lock file.
+        #
+        # @return [Hash]
+        def options_to_lock
+          {}
+        end
+
+        # Install the gem specified by the spec at appropriate path.
+        # `install_path` provides a sufficient default, if the source can only
+        # satisfy one gem,  but is not binding.
+        #
+        # @return [String] post installation message (if any)
+        def install(spec, opts)
+          raise MalformattedPlugin, "Source plugins need to override the install method."
+        end
+
+        # It builds extensions, generates bins and installs them for the spec
+        # provided.
+        #
+        # It depends on `spec.loaded_from` to get full_gem_path. The source
+        # plugins should set that.
+        #
+        # It should be called in `install` after the plugin is done placing the
+        # gem at correct install location.
+        #
+        # It also runs Gem hooks `post_install`, `post_build` and `post_install`
+        #
+        # Note: Do not override if you don't know what you are doing.
+        def post_install(spec, disable_exts = false)
+          opts = { :env_shebang => false, :disable_extensions => disable_exts }
+          installer = Bundler::Source::Path::Installer.new(spec, opts)
+          installer.post_install
+        end
+
+        # A default installation path to install a single gem. If the source
+        # servers multiple gems, it's not of much use and the source should one
+        # of its own.
+        def install_path
+          @install_path ||=
+            begin
+              base_name = File.basename(URI.parse(uri).normalize.path)
+
+              gem_install_dir.join("#{base_name}-#{uri_hash[0..11]}")
+            end
+        end
+
+        # Parses the gemspec files to find the specs for the gems that can be
+        # satisfied by the source.
+        #
+        # Few important points to keep in mind:
+        #   - If the gems are not installed then it shall return specs for all
+        #   the gems it can satisfy
+        #   - If gem is installed (that is to be detected by the plugin itself)
+        #   then it shall return at least the specs that are installed.
+        #   - The `loaded_from` for each of the specs shall be correct (it is
+        #   used to find the load path)
+        #
+        # @return [Bundler::Index] index containing the specs
+        def specs
+          files = fetch_gemspec_files
+
+          Bundler::Index.build do |index|
+            files.each do |file|
+              next unless spec = Bundler.load_gemspec(file)
+              Bundler.rubygems.set_installed_by_version(spec)
+
+              spec.source = self
+              Bundler.rubygems.validate(spec)
+
+              index << spec
+            end
+          end
+        end
+
+        # Set internal representation to fetch the gems/specs from remote.
+        #
+        # When this is called, the source should try to fetch the specs and
+        # install from remote path.
+        def remote!
+        end
+
+        # Set internal representation to fetch the gems/specs from app cache.
+        #
+        # When this is called, the source should try to fetch the specs and
+        # install from the path provided by `app_cache_path`.
+        def cached!
+        end
+
+        # This is called to update the spec and installation.
+        #
+        # If the source plugin is loaded from lockfile or otherwise, it shall
+        # refresh the cache/specs (e.g. git sources can make a fresh clone).
+        def unlock!
+        end
+
+        # Name of directory where plugin the is expected to cache the gems when
+        # #cache is called.
+        #
+        # Also this name is matched against the directories in cache for pruning
+        #
+        # This is used by `app_cache_path`
+        def app_cache_dirname
+          base_name = File.basename(URI.parse(uri).normalize.path)
+          "#{base_name}-#{uri_hash}"
+        end
+
+        # This method is called while caching to save copy of the gems that the
+        # source can resolve to path provided by `app_cache_app`so that they can
+        # be reinstalled from the cache without querying the remote (i.e. an
+        # alternative to remote)
+        #
+        # This is stored with the app and source plugins should try to provide
+        # specs and install only from this cache when `cached!` is called.
+        #
+        # This cache is different from the internal caching that can be done
+        # at sub paths of `cache_path` (from API). This can be though as caching
+        # by bundler.
+        def cache(spec, custom_path = nil)
+          new_cache_path = app_cache_path(custom_path)
+
+          FileUtils.rm_rf(new_cache_path)
+          FileUtils.cp_r(install_path, new_cache_path)
+          FileUtils.touch(app_cache_path.join(".bundlecache"))
+        end
+
+        # This shall check if two source object represent the same source.
+        #
+        # The comparison shall take place only on the attribute that can be
+        # inferred from the options passed from Gemfile and not on attibutes
+        # that are used to pin down the gem to specific version (e.g. Git
+        # sources should compare on branch and tag but not on commit hash)
+        #
+        # The sources objects are constructed from Gemfile as well as from
+        # lockfile. To converge the sources, it is necessary that they match.
+        #
+        # The same applies for `eql?` and `hash`
+        def ==(other)
+          other.is_a?(self.class) && uri == other.uri
+        end
+
+        # When overriding `eql?` please preserve the behaviour as mentioned in
+        # docstring for `==` method.
+        alias_method :eql?, :==
+
+        # When overriding `hash` please preserve the behaviour as mentioned in
+        # docstring for `==` method, i.e. two methods equal by above comparison
+        # should have same hash.
+        def hash
+          [self.class, uri].hash
+        end
+
+        # A helper method, not necessary if not used internally.
+        def installed?
+          File.directory?(install_path)
+        end
+
+        # The full path where the plugin should cache the gem so that it can be
+        # installed latter.
+        #
+        # Note: Do not override if you don't know what you are doing.
+        def app_cache_path(custom_path = nil)
+          @app_cache_path ||= Bundler.app_cache(custom_path).join(app_cache_dirname)
+        end
+
+        # Used by definition.
+        #
+        # Note: Do not override if you don't know what you are doing.
+        def unmet_deps
+          specs.unmet_dependency_names
+        end
+
+        # Note: Do not override if you don't know what you are doing.
+        def can_lock?(spec)
+          spec.source == self
+        end
+
+        # Generates the content to be entered into the lockfile.
+        # Saves type and remote and also calls to `options_to_lock`.
+        #
+        # Plugin should use `options_to_lock` to save information in lockfile
+        # and not override this.
+        #
+        # Note: Do not override if you don't know what you are doing.
+        def to_lock
+          out = String.new("#{LockfileParser::PLUGIN}\n")
+          out << "  remote: #{@uri}\n"
+          out << "  type: #{@type}\n"
+          options_to_lock.each do |opt, value|
+            out << "  #{opt}: #{value}\n"
+          end
+          out << "  specs:\n"
+        end
+
+        def to_s
+          "plugin source for #{options[:type]} with uri #{uri}"
+        end
+
+        # Note: Do not override if you don't know what you are doing.
+        def include?(other)
+          other == self
+        end
+
+        def uri_hash
+          Digest::SHA1.hexdigest(uri)
+        end
+
+        # Note: Do not override if you don't know what you are doing.
+        def gem_install_dir
+          Bundler.install_path
+        end
+
+        # It is used to obtain the full_gem_path.
+        #
+        # spec's loaded_from path is expanded against this to get full_gem_path
+        #
+        # Note: Do not override if you don't know what you are doing.
+        def root
+          Bundler.root
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bundler/plugin/dsl.rb b/lib/bundler/plugin/dsl.rb
index f65054f0..4bfc8437 100644
--- a/lib/bundler/plugin/dsl.rb
+++ b/lib/bundler/plugin/dsl.rb
@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
 module Bundler
-  # Dsl to parse the Gemfile looking for plugins to install
   module Plugin
+    # Dsl to parse the Gemfile looking for plugins to install
     class DSL < Bundler::Dsl
       class PluginGemfileError < PluginError; end
       alias_method :_gem, :gem # To use for plugin installation as gem
@@ -12,9 +12,20 @@ module Bundler
       # They will be handled by method_missing
       [:gemspec, :gem, :path, :install_if, :platforms, :env].each {|m| undef_method m }
 
+      # This lists the plugins that was added automatically and not specified by
+      # the user.
+      #
+      # When we encounter :type attribute with a source block, we add a plugin
+      # by name bundler-source-<type> to list of plugins to be installed.
+      #
+      # These plugins are optional and are not installed when there is conflict
+      # with any other plugin.
+      attr_reader :inferred_plugins
+
       def initialize
         super
         @sources = Plugin::SourceList.new
+        @inferred_plugins = [] # The source plugins inferred from :type
       end
 
       def plugin(name, *args)
@@ -24,6 +35,19 @@ module Bundler
       def method_missing(name, *args)
         raise PluginGemfileError, "Undefined local variable or method `#{name}' for Gemfile" unless Bundler::Dsl.method_defined? name
       end
+
+      def source(source, *args, &blk)
+        options = args.last.is_a?(Hash) ? args.pop.dup : {}
+        options = normalize_hash(options)
+        return super unless options.key?("type")
+
+        plugin_name = "bundler-source-#{options["type"]}"
+
+        return if @dependencies.any? {|d| d.name == plugin_name }
+
+        plugin(plugin_name)
+        @inferred_plugins << plugin_name
+      end
     end
   end
 end
diff --git a/lib/bundler/plugin/index.rb b/lib/bundler/plugin/index.rb
index 1e39eb00..4abf85fb 100644
--- a/lib/bundler/plugin/index.rb
+++ b/lib/bundler/plugin/index.rb
@@ -13,27 +13,48 @@ module Bundler
         end
       end
 
+      class SourceConflict < PluginError
+        def initialize(plugin, sources)
+          msg = "Source(s) `#{sources.join("`, `")}` declared by #{plugin} are already registered."
+          super msg
+        end
+      end
+
       def initialize
         @plugin_paths = {}
         @commands = {}
+        @sources = {}
+        @load_paths = {}
 
         load_index
       end
 
-      # This function is to be called when a new plugin is installed. This function shall add
-      # the functions of the plugin to existing maps and also the name to source location.
+      # This function is to be called when a new plugin is installed. This
+      # function shall add the functions of the plugin to existing maps and also
+      # the name to source location.
       #
       # @param [String] name of the plugin to be registered
       # @param [String] path where the plugin is installed
+      # @param [Array<String>] load_paths for the plugin
       # @param [Array<String>] commands that are handled by the plugin
-      def register_plugin(name, path, commands)
-        @plugin_paths[name] = path
+      # @param [Array<String>] sources that are handled by the plugin
+      def register_plugin(name, path, load_paths, commands, sources)
+        old_commands = @commands.dup
 
         common = commands & @commands.keys
         raise CommandConflict.new(name, common) unless common.empty?
         commands.each {|c| @commands[c] = name }
 
+        common = sources & @sources.keys
+        raise SourceConflict.new(name, common) unless common.empty?
+        sources.each {|k| @sources[k] = name }
+
+        @plugin_paths[name] = path
+        @load_paths[name] = load_paths
         save_index
+      rescue
+        @commands = old_commands
+        raise
       end
 
       # Path where the index file is stored
@@ -45,6 +66,10 @@ module Bundler
         Pathname.new @plugin_paths[name]
       end
 
+      def load_paths(name)
+        @load_paths[name]
+      end
+
       # Fetch the name of plugin handling the command
       def command_plugin(command)
         @commands[command]
@@ -54,9 +79,18 @@ module Bundler
         @plugin_paths[name]
       end
 
+      def source?(source)
+        @sources.key? source
+      end
+
+      def source_plugin(name)
+        @sources[name]
+      end
+
     private
 
-      # Reads the index file from the directory and initializes the instance variables.
+      # Reads the index file from the directory and initializes the instance
+      # variables.
       def load_index
         SharedHelpers.filesystem_access(index_file, :read) do |index_f|
           valid_file = index_f && index_f.exist? && !index_f.size.zero?
@@ -65,16 +99,21 @@ module Bundler
           require "bundler/yaml_serializer"
           index = YAMLSerializer.load(data)
           @plugin_paths = index["plugin_paths"] || {}
+          @load_paths = index["load_paths"] || {}
           @commands = index["commands"] || {}
+          @sources = index["sources"] || {}
         end
       end
 
-      # Should be called when any of the instance variables change. Stores the instance
-      # variables in YAML format. (The instance variables are supposed to be only String key value pairs)
+      # Should be called when any of the instance variables change. Stores the
+      # instance variables in YAML format. (The instance variables are supposed
+      # to be only String key value pairs)
       def save_index
         index = {
           "plugin_paths" => @plugin_paths,
+          "load_paths" => @load_paths,
           "commands" => @commands,
+          "sources" => @sources,
         }
 
         require "bundler/yaml_serializer"
diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb
index 2c10bb24..a50d0cee 100644
--- a/lib/bundler/plugin/installer.rb
+++ b/lib/bundler/plugin/installer.rb
@@ -25,18 +25,14 @@ module Bundler
       # Installs the plugin from Definition object created by limited parsing of
       # Gemfile searching for plugins to be installed
       #
-      # @param [Definition] definiton object
-      # @return [Hash] map of plugin names to thier paths
+      # @param [Definition] definition object
+      # @return [Hash] map of names to their specs they are installed with
       def install_definition(definition)
-        plugins = definition.dependencies.map(&:name)
-
         def definition.lock(*); end
         definition.resolve_remotely!
         specs = definition.specs
 
-        paths = install_from_specs specs
-
-        Hash[paths.select {|name, _| plugins.include? name }]
+        install_from_specs specs
       end
 
     private
@@ -66,7 +62,7 @@ module Bundler
       # @param [Array] version of the gem to install
       # @param [String, Array<String>] source(s) to resolve the gem
       #
-      # @return [String] the path where the plugin was installed
+      # @return [Hash] map of names to the specs of plugins installed
       def install_rubygems(names, version, sources)
         deps = names.map {|name| Dependency.new name, version }
 
@@ -82,14 +78,14 @@ module Bundler
       #
       # @param specs to install
       #
-      # @return [Hash] map of names to path where the plugin was installed
+      # @return [Hash] map of names to the specs
       def install_from_specs(specs)
         paths = {}
 
         specs.each do |spec|
           spec.source.install spec
 
-          paths[spec.name] = spec.full_gem_path
+          paths[spec.name] = spec
         end
 
         paths
diff --git a/lib/bundler/plugin/source_list.rb b/lib/bundler/plugin/source_list.rb
index 6b1f1aee..33f5e5af 100644
--- a/lib/bundler/plugin/source_list.rb
+++ b/lib/bundler/plugin/source_list.rb
@@ -19,6 +19,10 @@ module Bundler
       def add_rubygems_source(options = {})
         add_source_to_list Plugin::Installer::Rubygems.new(options), @rubygems_sources
       end
+
+      def all_sources
+        path_sources + git_sources + rubygems_sources
+      end
     end
   end
 end
diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb
index 53db29a9..88c446e1 100644
--- a/lib/bundler/rubygems_ext.rb
+++ b/lib/bundler/rubygems_ext.rb
@@ -25,7 +25,7 @@ module Gem
     attr_writer :full_gem_path unless instance_methods.include?(:full_gem_path=)
 
     def full_gem_path
-      if source.respond_to?(:path)
+      if source.respond_to?(:path) || source.is_a?(Bundler::Plugin::API::Source)
         Pathname.new(loaded_from).dirname.expand_path(source.root).to_s.untaint
       else
         rg_full_gem_path
diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb
index 76955ee4..5344ab69 100644
--- a/lib/bundler/source/git.rb
+++ b/lib/bundler/source/git.rb
@@ -170,7 +170,7 @@ module Bundler
           serialize_gemspecs_in(install_path)
           @copied = true
         end
-        generate_bin(spec)
+        generate_bin(spec, !Bundler.rubygems.spec_missing_extensions?(spec))
 
         requires_checkout? ? spec.post_install_message : nil
       end
@@ -223,10 +223,6 @@ module Bundler
 
     private
 
-      def build_extensions(installer)
-        super if Bundler.rubygems.spec_missing_extensions?(installer.spec)
-      end
-
       def serialize_gemspecs_in(destination)
         destination = destination.expand_path(Bundler.root) if destination.relative?
         Dir["#{destination}/#{@glob}"].each do |spec_path|
diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb
index bbdd30b1..3c4d914f 100644
--- a/lib/bundler/source/path.rb
+++ b/lib/bundler/source/path.rb
@@ -203,13 +203,8 @@ module Bundler
           end
         end.compact
 
-        SharedHelpers.chdir(gem_dir) do
-          installer = Path::Installer.new(spec, :env_shebang => false)
-          run_hooks(:pre_install, installer)
-          build_extensions(installer) unless disable_extensions
-          installer.generate_bin
-          run_hooks(:post_install, installer)
-        end
+        installer = Path::Installer.new(spec, :env_shebang => false, :disable_extensions => disable_extensions)
+        installer.post_install
       rescue Gem::InvalidSpecificationException => e
         Bundler.ui.warn "\n#{spec.name} at #{spec.full_gem_path} did not have a valid gemspec.\n" \
                         "This prevents bundler from installing bins or native extensions, but " \
@@ -223,23 +218,6 @@ module Bundler
 
         Bundler.ui.warn "The validation message from Rubygems was:\n  #{e.message}"
       end
-
-      def build_extensions(installer)
-        installer.build_extensions
-        run_hooks(:post_build, installer)
-      end
-
-      def run_hooks(type, installer)
-        hooks_meth = "#{type}_hooks"
-        return unless Gem.respond_to?(hooks_meth)
-        Gem.send(hooks_meth).each do |hook|
-          result = hook.call(installer)
-          next unless result == false
-          location = " at #{$1}" if hook.inspect =~ /@(.*:\d+)/
-          message = "#{type} hook#{location} failed for #{installer.spec.full_name}"
-          raise InstallHookError, message
-        end
-      end
     end
   end
 end
diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb
index 0aa27bd5..abc46d5a 100644
--- a/lib/bundler/source/path/installer.rb
+++ b/lib/bundler/source/path/installer.rb
@@ -6,13 +6,14 @@ module Bundler
         attr_reader :spec
 
         def initialize(spec, options = {})
-          @spec              = spec
-          @gem_dir           = Bundler.rubygems.path(spec.full_gem_path)
-          @wrappers          = true
-          @env_shebang       = true
-          @format_executable = options[:format_executable] || false
-          @build_args        = options[:build_args] || Bundler.rubygems.build_args
-          @gem_bin_dir       = "#{Bundler.rubygems.gem_dir}/bin"
+          @spec               = spec
+          @gem_dir            = Bundler.rubygems.path(spec.full_gem_path)
+          @wrappers           = true
+          @env_shebang        = true
+          @format_executable  = options[:format_executable] || false
+          @build_args         = options[:build_args] || Bundler.rubygems.build_args
+          @gem_bin_dir        = "#{Bundler.rubygems.gem_dir}/bin"
+          @disable_extentions = options[:disable_extensions]
 
           if Bundler.requires_sudo?
             @tmp_dir = Bundler.tmp(spec.full_name).to_s
@@ -22,9 +23,26 @@ module Bundler
           end
         end
 
-        def generate_bin
-          return if spec.executables.nil? || spec.executables.empty?
+        def post_install
+          SharedHelpers.chdir(@gem_dir) do
+            run_hooks(:pre_install)
+
+            unless @disable_extentions
+              build_extensions
+              run_hooks(:post_build)
+            end
+
+            generate_bin unless spec.executables.nil? || spec.executables.empty?
+
+            run_hooks(:post_install)
+          end
+        ensure
+          Bundler.rm_rf(@tmp_dir) if Bundler.requires_sudo?
+        end
+
+      private
 
+        def generate_bin
           super
 
           if Bundler.requires_sudo?
@@ -35,8 +53,18 @@ module Bundler
               Bundler.sudo "cp -R #{@bin_dir}/#{exe} #{@gem_bin_dir}"
             end
           end
-        ensure
-          Bundler.rm_rf(@tmp_dir) if Bundler.requires_sudo?
+        end
+
+        def run_hooks(type)
+          hooks_meth = "#{type}_hooks"
+          return unless Gem.respond_to?(hooks_meth)
+          Gem.send(hooks_meth).each do |hook|
+            result = hook.call(self)
+            next unless result == false
+            location = " at #{$1}" if hook.inspect =~ /@(.*:\d+)/
+            message = "#{type} hook#{location} failed for #{spec.full_name}"
+            raise InstallHookError, message
+          end
         end
       end
     end
diff --git a/lib/bundler/source_list.rb b/lib/bundler/source_list.rb
index 0595cbdc..fdc77cb2 100644
--- a/lib/bundler/source_list.rb
+++ b/lib/bundler/source_list.rb
@@ -2,11 +2,13 @@
 module Bundler
   class SourceList
     attr_reader :path_sources,
-      :git_sources
+      :git_sources,
+      :plugin_sources
 
     def initialize
       @path_sources       = []
       @git_sources        = []
+      @plugin_sources     = []
       @rubygems_aggregate = Source::Rubygems.new
       @rubygems_sources   = []
     end
@@ -27,6 +29,10 @@ module Bundler
       add_source_to_list Source::Rubygems.new(options), @rubygems_sources
     end
 
+    def add_plugin_source(source, options = {})
+      add_source_to_list Plugin.source(source).new(options), @plugin_sources
+    end
+
     def add_rubygems_remote(uri)
       @rubygems_aggregate.add_remote(uri)
       @rubygems_aggregate
@@ -41,7 +47,7 @@ module Bundler
     end
 
     def all_sources
-      path_sources + git_sources + rubygems_sources
+      path_sources + git_sources + plugin_sources + rubygems_sources
     end
 
     def get(source)
@@ -49,14 +55,14 @@ module Bundler
     end
 
     def lock_sources
-      lock_sources = (path_sources + git_sources).sort_by(&:to_s)
+      lock_sources = (path_sources + git_sources + plugin_sources).sort_by(&:to_s)
       lock_sources << combine_rubygems_sources
     end
 
     def replace_sources!(replacement_sources)
       return true if replacement_sources.empty?
 
-      [path_sources, git_sources].each do |source_list|
+      [path_sources, git_sources, plugin_sources].each do |source_list|
         source_list.map! do |source|
           replacement_sources.find {|s| s == source } || source
         end
@@ -92,9 +98,10 @@ module Bundler
 
     def source_list_for(source)
       case source
-      when Source::Git      then git_sources
-      when Source::Path     then path_sources
-      when Source::Rubygems then rubygems_sources
+      when Source::Git          then git_sources
+      when Source::Path         then path_sources
+      when Source::Rubygems     then rubygems_sources
+      when Plugin::API::Source  then plugin_sources
       else raise ArgumentError, "Invalid source: #{source.inspect}"
       end
     end
diff --git a/lib/bundler/yaml_serializer.rb b/lib/bundler/yaml_serializer.rb
index 327baa4e..dede8fd5 100644
--- a/lib/bundler/yaml_serializer.rb
+++ b/lib/bundler/yaml_serializer.rb
@@ -16,6 +16,8 @@ module Bundler
         yaml << k << ":"
         if v.is_a?(Hash)
           yaml << dump_hash(v).gsub(/^(?!$)/, "  ") # indent all non-empty lines
+        elsif v.is_a?(Array) # Expected to be array of strings
+          yaml << "\n- " << v.map {|s| s.to_s.gsub(/\s+/, " ").inspect }.join("\n- ") << "\n"
         else
           yaml << " " << v.to_s.gsub(/\s+/, " ").inspect << "\n"
         end
@@ -23,11 +25,20 @@ module Bundler
       yaml
     end
 
-    SCAN_REGEX = /
+    ARRAY_REGEX = /
+      ^
+      (?:[ ]*-[ ]) # '- ' before array items
+      (['"]?) # optional opening quote
+      (.*) # value
+      \1 # matching closing quote
+      $
+    /xo
+
+    HASH_REGEX = /
       ^
       ([ ]*) # indentations
       (.*) # key
-      (?::(?=\s)) # :  (without the lookahead the #key includes this when : is present in value)
+      (?::(?=(?:\s|$))) # :  (without the lookahead the #key includes this when : is present in value)
       [ ]?
       (?: !\s)? # optional exclamation mark found with ruby 1.9.3
       (['"]?) # optional opening quote
@@ -39,15 +50,27 @@ module Bundler
     def load(str)
       res = {}
       stack = [res]
-      str.scan(SCAN_REGEX).each do |(indent, key, _, val)|
-        key = convert_to_backward_compatible_key(key)
-        depth = indent.scan(/  /).length
-        if val.empty?
-          new_hash = {}
-          stack[depth][key] = new_hash
-          stack[depth + 1] = new_hash
-        else
-          stack[depth][key] = val
+      last_hash = nil
+      last_empty_key = nil
+      str.split("\n").each do |line|
+        if match = HASH_REGEX.match(line)
+          indent, key, _, val = match.captures
+          key = convert_to_backward_compatible_key(key)
+          depth = indent.scan(/  /).length
+          if val.empty?
+            new_hash = {}
+            stack[depth][key] = new_hash
+            stack[depth + 1] = new_hash
+            last_empty_key = key
+            last_hash = stack[depth]
+          else
+            stack[depth][key] = val
+          end
+        elsif match = ARRAY_REGEX.match(line)
+          _, val = match.captures
+          last_hash[last_empty_key] = [] unless last_hash[last_empty_key].is_a?(Array)
+
+          last_hash[last_empty_key].push(val)
         end
       end
       res