From 8598f8c2dc78c6d1ae87cb6ae19c34ba2cb29241 Mon Sep 17 00:00:00 2001 From: hsbt Date: Fri, 8 Sep 2017 08:45:41 +0000 Subject: Merge bundler to standard libraries. rubygems 2.7.x depends bundler-1.15.x. This is preparation for rubygems and bundler migration. * lib/bundler.rb, lib/bundler/*: files of bundler-1.15.4 * spec/bundler/*: rspec examples of bundler-1.15.4. I applied patches. * https://github.com/bundler/bundler/pull/6007 * Exclude not working examples on ruby repository. * Fake ruby interpriter instead of installed ruby. * Makefile.in: Added test task named `test-bundler`. This task is only working macOS/linux yet. I'm going to support Windows environment later. * tool/sync_default_gems.rb: Added sync task for bundler. [Feature #12733][ruby-core:77172] git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59779 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- lib/bundler/plugin/api.rb | 81 +++++++++ lib/bundler/plugin/api/source.rb | 299 +++++++++++++++++++++++++++++++ lib/bundler/plugin/dsl.rb | 53 ++++++ lib/bundler/plugin/index.rb | 157 ++++++++++++++++ lib/bundler/plugin/installer.rb | 95 ++++++++++ lib/bundler/plugin/installer/git.rb | 38 ++++ lib/bundler/plugin/installer/rubygems.rb | 27 +++ lib/bundler/plugin/source_list.rb | 28 +++ 8 files changed, 778 insertions(+) create mode 100644 lib/bundler/plugin/api.rb create mode 100644 lib/bundler/plugin/api/source.rb create mode 100644 lib/bundler/plugin/dsl.rb create mode 100644 lib/bundler/plugin/index.rb create mode 100644 lib/bundler/plugin/installer.rb create mode 100644 lib/bundler/plugin/installer/git.rb create mode 100644 lib/bundler/plugin/installer/rubygems.rb create mode 100644 lib/bundler/plugin/source_list.rb (limited to 'lib/bundler/plugin') diff --git a/lib/bundler/plugin/api.rb b/lib/bundler/plugin/api.rb new file mode 100644 index 0000000000..a2d5cbb4ac --- /dev/null +++ b/lib/bundler/plugin/api.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Bundler + # This is the interfacing class represents the API that we intend to provide + # the plugins to use. + # + # For plugins to be independent of the Bundler internals they shall limit their + # interactions to methods of this class only. This will save them from breaking + # when some internal change. + # + # Currently we are delegating the methods defined in Bundler class to + # itself. So, this class acts as a buffer. + # + # If there is some change in the Bundler class that is incompatible to its + # previous behavior or if otherwise desired, we can reimplement(or implement) + # the method to preserve compatibility. + # + # To use this, either the class can inherit this class or use it directly. + # For example of both types of use, refer the file `spec/plugins/command.rb` + # + # To use it without inheriting, you will have to create an object of this + # to use the functions (except for declaration functions like command, source, + # and hooks). + module Plugin + class API + autoload :Source, "bundler/plugin/api/source" + + # The plugins should declare that they handle a command through this helper. + # + # @param [String] command being handled by them + # @param [Class] (optional) class that handles the command. If not + # provided, the `self` class will be used. + def self.command(command, cls = self) + Plugin.add_command command, cls + end + + # The plugins should declare that they provide a installation source + # through this helper. + # + # @param [String] the source type they provide + # @param [Class] (optional) class that handles the source. If not + # provided, the `self` class will be used. + def self.source(source, cls = self) + cls.send :include, Bundler::Plugin::API::Source + Plugin.add_source source, cls + end + + def self.hook(event, &block) + Plugin.add_hook(event, &block) + end + + # The cache dir to be used by the plugins for storage + # + # @return [Pathname] path of the cache dir + def cache_dir + Plugin.cache.join("plugins") + end + + # A tmp dir to be used by plugins + # Accepts names that get concatenated as suffix + # + # @return [Pathname] object for the new directory created + def tmp(*names) + Bundler.tmp(["plugin", *names].join("-")) + end + + def method_missing(name, *args, &blk) + return Bundler.send(name, *args, &blk) if Bundler.respond_to?(name) + + return SharedHelpers.send(name, *args, &blk) if SharedHelpers.respond_to?(name) + + super + end + + def respond_to_missing?(name, include_private = false) + SharedHelpers.respond_to?(name, include_private) || + Bundler.respond_to?(name, include_private) || super + end + end + end +end diff --git a/lib/bundler/plugin/api/source.rb b/lib/bundler/plugin/api/source.rb new file mode 100644 index 0000000000..5d3f58df92 --- /dev/null +++ b/lib/bundler/plugin/api/source.rb @@ -0,0 +1,299 @@ +# frozen_string_literal: true +require "uri" +require "digest/sha1" + +module Bundler + module Plugin + class API + # This class provides the base to build source plugins + # All the method here are required to build a source plugin (except + # `uri_hash`, `gem_install_dir`; they are helpers). + # + # Defaults for methods, where ever possible are provided which is + # expected to work. But, all source plugins have to override + # `fetch_gemspec_files` and `install`. Defaults are also not provided for + # `remote!`, `cache!` and `unlock!`. + # + # The defaults shall work for most situations but nevertheless they can + # be (preferably should be) overridden as per the plugins' needs safely + # (as long as they behave as expected). + # On overriding `initialize` you should call super first. + # + # If required plugin should override `hash`, `==` and `eql?` methods to be + # able to match objects representing same sources, but may be created in + # different situation (like form gemfile and lockfile). The default ones + # checks only for class and uri, but elaborate source plugins may need + # more comparisons (e.g. git checking on branch or tag). + # + # @!attribute [r] uri + # @return [String] the remote specified with `source` block in Gemfile + # + # @!attribute [r] options + # @return [String] options passed during initialization (either from + # lockfile or Gemfile) + # + # @!attribute [r] name + # @return [String] name that can be used to uniquely identify a source + # + # @!attribute [rw] dependency_names + # @return [Array] Names of dependencies that the source should + # try to resolve. It is not necessary to use this list intenally. This + # is present to be compatible with `Definition` and is used by + # rubygems source. + module Source + attr_reader :uri, :options, :name + attr_accessor :dependency_names + + def initialize(opts) + @options = opts + @dependency_names = [] + @uri = opts["uri"] + @type = opts["type"] + @name = opts["name"] || "#{@type} at #{@uri}" + end + + # This is used by the default `spec` method to constructs the + # Specification objects for the gems and versions that can be installed + # by this source plugin. + # + # Note: If the spec method is overridden, this function is not necessary + # + # @return [Array] paths of the gemspec files for gems that can + # be installed + def fetch_gemspec_files + [] + end + + # Options to be saved in the lockfile so that the source plugin is able + # to check out same version of gem later. + # + # There options are passed when the source plugin is created from the + # lock file. + # + # @return [Hash] + def options_to_lock + {} + end + + # Install the gem specified by the spec at appropriate path. + # `install_path` provides a sufficient default, if the source can only + # satisfy one gem, but is not binding. + # + # @return [String] post installation message (if any) + def install(spec, opts) + raise MalformattedPlugin, "Source plugins need to override the install method." + end + + # It builds extensions, generates bins and installs them for the spec + # provided. + # + # It depends on `spec.loaded_from` to get full_gem_path. The source + # plugins should set that. + # + # It should be called in `install` after the plugin is done placing the + # gem at correct install location. + # + # It also runs Gem hooks `pre_install`, `post_build` and `post_install` + # + # Note: Do not override if you don't know what you are doing. + def post_install(spec, disable_exts = false) + opts = { :env_shebang => false, :disable_extensions => disable_exts } + installer = Bundler::Source::Path::Installer.new(spec, opts) + installer.post_install + end + + # A default installation path to install a single gem. If the source + # servers multiple gems, it's not of much use and the source should one + # of its own. + def install_path + @install_path ||= + begin + base_name = File.basename(URI.parse(uri).normalize.path) + + gem_install_dir.join("#{base_name}-#{uri_hash[0..11]}") + end + end + + # Parses the gemspec files to find the specs for the gems that can be + # satisfied by the source. + # + # Few important points to keep in mind: + # - If the gems are not installed then it shall return specs for all + # the gems it can satisfy + # - If gem is installed (that is to be detected by the plugin itself) + # then it shall return at least the specs that are installed. + # - The `loaded_from` for each of the specs shall be correct (it is + # used to find the load path) + # + # @return [Bundler::Index] index containing the specs + def specs + files = fetch_gemspec_files + + Bundler::Index.build do |index| + files.each do |file| + next unless spec = Bundler.load_gemspec(file) + Bundler.rubygems.set_installed_by_version(spec) + + spec.source = self + Bundler.rubygems.validate(spec) + + index << spec + end + end + end + + # Set internal representation to fetch the gems/specs from remote. + # + # When this is called, the source should try to fetch the specs and + # install from remote path. + def remote! + end + + # Set internal representation to fetch the gems/specs from app cache. + # + # When this is called, the source should try to fetch the specs and + # install from the path provided by `app_cache_path`. + def cached! + end + + # This is called to update the spec and installation. + # + # If the source plugin is loaded from lockfile or otherwise, it shall + # refresh the cache/specs (e.g. git sources can make a fresh clone). + def unlock! + end + + # Name of directory where plugin the is expected to cache the gems when + # #cache is called. + # + # Also this name is matched against the directories in cache for pruning + # + # This is used by `app_cache_path` + def app_cache_dirname + base_name = File.basename(URI.parse(uri).normalize.path) + "#{base_name}-#{uri_hash}" + end + + # This method is called while caching to save copy of the gems that the + # source can resolve to path provided by `app_cache_app`so that they can + # be reinstalled from the cache without querying the remote (i.e. an + # alternative to remote) + # + # This is stored with the app and source plugins should try to provide + # specs and install only from this cache when `cached!` is called. + # + # This cache is different from the internal caching that can be done + # at sub paths of `cache_path` (from API). This can be though as caching + # by bundler. + def cache(spec, custom_path = nil) + new_cache_path = app_cache_path(custom_path) + + FileUtils.rm_rf(new_cache_path) + FileUtils.cp_r(install_path, new_cache_path) + FileUtils.touch(app_cache_path.join(".bundlecache")) + end + + # This shall check if two source object represent the same source. + # + # The comparison shall take place only on the attribute that can be + # inferred from the options passed from Gemfile and not on attibutes + # that are used to pin down the gem to specific version (e.g. Git + # sources should compare on branch and tag but not on commit hash) + # + # The sources objects are constructed from Gemfile as well as from + # lockfile. To converge the sources, it is necessary that they match. + # + # The same applies for `eql?` and `hash` + def ==(other) + other.is_a?(self.class) && uri == other.uri + end + + # When overriding `eql?` please preserve the behaviour as mentioned in + # docstring for `==` method. + alias_method :eql?, :== + + # When overriding `hash` please preserve the behaviour as mentioned in + # docstring for `==` method, i.e. two methods equal by above comparison + # should have same hash. + def hash + [self.class, uri].hash + end + + # A helper method, not necessary if not used internally. + def installed? + File.directory?(install_path) + end + + # The full path where the plugin should cache the gem so that it can be + # installed latter. + # + # Note: Do not override if you don't know what you are doing. + def app_cache_path(custom_path = nil) + @app_cache_path ||= Bundler.app_cache(custom_path).join(app_cache_dirname) + end + + # Used by definition. + # + # Note: Do not override if you don't know what you are doing. + def unmet_deps + specs.unmet_dependency_names + end + + # Note: Do not override if you don't know what you are doing. + def can_lock?(spec) + spec.source == self + end + + # Generates the content to be entered into the lockfile. + # Saves type and remote and also calls to `options_to_lock`. + # + # Plugin should use `options_to_lock` to save information in lockfile + # and not override this. + # + # Note: Do not override if you don't know what you are doing. + def to_lock + out = String.new("#{LockfileParser::PLUGIN}\n") + out << " remote: #{@uri}\n" + out << " type: #{@type}\n" + options_to_lock.each do |opt, value| + out << " #{opt}: #{value}\n" + end + out << " specs:\n" + end + + def to_s + "plugin source for #{options[:type]} with uri #{uri}" + end + + # Note: Do not override if you don't know what you are doing. + def include?(other) + other == self + end + + def uri_hash + Digest::SHA1.hexdigest(uri) + end + + # Note: Do not override if you don't know what you are doing. + def gem_install_dir + Bundler.install_path + end + + # It is used to obtain the full_gem_path. + # + # spec's loaded_from path is expanded against this to get full_gem_path + # + # Note: Do not override if you don't know what you are doing. + def root + Bundler.root + end + + # @private + # Returns true + def bundler_plugin_api_source? + true + end + end + end + end +end diff --git a/lib/bundler/plugin/dsl.rb b/lib/bundler/plugin/dsl.rb new file mode 100644 index 0000000000..4bfc8437e0 --- /dev/null +++ b/lib/bundler/plugin/dsl.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + # Dsl to parse the Gemfile looking for plugins to install + class DSL < Bundler::Dsl + class PluginGemfileError < PluginError; end + alias_method :_gem, :gem # To use for plugin installation as gem + + # So that we don't have to override all there methods to dummy ones + # explicitly. + # They will be handled by method_missing + [:gemspec, :gem, :path, :install_if, :platforms, :env].each {|m| undef_method m } + + # This lists the plugins that was added automatically and not specified by + # the user. + # + # When we encounter :type attribute with a source block, we add a plugin + # by name bundler-source- to list of plugins to be installed. + # + # These plugins are optional and are not installed when there is conflict + # with any other plugin. + attr_reader :inferred_plugins + + def initialize + super + @sources = Plugin::SourceList.new + @inferred_plugins = [] # The source plugins inferred from :type + end + + def plugin(name, *args) + _gem(name, *args) + end + + def method_missing(name, *args) + raise PluginGemfileError, "Undefined local variable or method `#{name}' for Gemfile" unless Bundler::Dsl.method_defined? name + end + + def source(source, *args, &blk) + options = args.last.is_a?(Hash) ? args.pop.dup : {} + options = normalize_hash(options) + return super unless options.key?("type") + + plugin_name = "bundler-source-#{options["type"]}" + + return if @dependencies.any? {|d| d.name == plugin_name } + + plugin(plugin_name) + @inferred_plugins << plugin_name + end + end + end +end diff --git a/lib/bundler/plugin/index.rb b/lib/bundler/plugin/index.rb new file mode 100644 index 0000000000..8dde072f16 --- /dev/null +++ b/lib/bundler/plugin/index.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true + +module Bundler + # Manages which plugins are installed and their sources. This also is supposed to map + # which plugin does what (currently the features are not implemented so this class is + # now a stub class). + module Plugin + class Index + class CommandConflict < PluginError + def initialize(plugin, commands) + msg = "Command(s) `#{commands.join("`, `")}` declared by #{plugin} are already registered." + super msg + end + end + + class SourceConflict < PluginError + def initialize(plugin, sources) + msg = "Source(s) `#{sources.join("`, `")}` declared by #{plugin} are already registered." + super msg + end + end + + attr_reader :commands + + def initialize + @plugin_paths = {} + @commands = {} + @sources = {} + @hooks = {} + @load_paths = {} + + load_index(global_index_file, true) + load_index(local_index_file) if SharedHelpers.in_bundle? + end + + # This function is to be called when a new plugin is installed. This + # function shall add the functions of the plugin to existing maps and also + # the name to source location. + # + # @param [String] name of the plugin to be registered + # @param [String] path where the plugin is installed + # @param [Array] load_paths for the plugin + # @param [Array] commands that are handled by the plugin + # @param [Array] sources that are handled by the plugin + def register_plugin(name, path, load_paths, commands, sources, hooks) + old_commands = @commands.dup + + common = commands & @commands.keys + raise CommandConflict.new(name, common) unless common.empty? + commands.each {|c| @commands[c] = name } + + common = sources & @sources.keys + raise SourceConflict.new(name, common) unless common.empty? + sources.each {|k| @sources[k] = name } + + hooks.each {|e| (@hooks[e] ||= []) << name } + + @plugin_paths[name] = path + @load_paths[name] = load_paths + save_index + rescue + @commands = old_commands + raise + end + + # Path of default index file + def index_file + Plugin.root.join("index") + end + + # Path where the global index file is stored + def global_index_file + Plugin.global_root.join("index") + end + + # Path where the local index file is stored + def local_index_file + Plugin.local_root.join("index") + end + + def plugin_path(name) + Pathname.new @plugin_paths[name] + end + + def load_paths(name) + @load_paths[name] + end + + # Fetch the name of plugin handling the command + def command_plugin(command) + @commands[command] + end + + def installed?(name) + @plugin_paths[name] + end + + def source?(source) + @sources.key? source + end + + def source_plugin(name) + @sources[name] + end + + # Returns the list of plugin names handling the passed event + def hook_plugins(event) + @hooks[event] || [] + end + + private + + # Reads the index file from the directory and initializes the instance + # variables. + # + # It skips the sources if the second param is true + # @param [Pathname] index file path + # @param [Boolean] is the index file global index + def load_index(index_file, global = false) + SharedHelpers.filesystem_access(index_file, :read) do |index_f| + valid_file = index_f && index_f.exist? && !index_f.size.zero? + break unless valid_file + + data = index_f.read + + require "bundler/yaml_serializer" + index = YAMLSerializer.load(data) + + @commands.merge!(index["commands"]) + @hooks.merge!(index["hooks"]) + @load_paths.merge!(index["load_paths"]) + @plugin_paths.merge!(index["plugin_paths"]) + @sources.merge!(index["sources"]) unless global + end + end + + # Should be called when any of the instance variables change. Stores the + # instance variables in YAML format. (The instance variables are supposed + # to be only String key value pairs) + def save_index + index = { + "commands" => @commands, + "hooks" => @hooks, + "load_paths" => @load_paths, + "plugin_paths" => @plugin_paths, + "sources" => @sources, + } + + require "bundler/yaml_serializer" + SharedHelpers.filesystem_access(index_file) do |index_f| + FileUtils.mkdir_p(index_f.dirname) + File.open(index_f, "w") {|f| f.puts YAMLSerializer.dump(index) } + end + end + end + end +end diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb new file mode 100644 index 0000000000..a50d0ceedd --- /dev/null +++ b/lib/bundler/plugin/installer.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Bundler + # Handles the installation of plugin in appropriate directories. + # + # This class is supposed to be wrapper over the existing gem installation infra + # but currently it itself handles everything as the Source's subclasses (e.g. Source::RubyGems) + # are heavily dependent on the Gemfile. + module Plugin + class Installer + autoload :Rubygems, "bundler/plugin/installer/rubygems" + autoload :Git, "bundler/plugin/installer/git" + + def install(names, options) + version = options[:version] || [">= 0"] + + if options[:git] + install_git(names, version, options) + else + sources = options[:source] || Bundler.rubygems.sources + install_rubygems(names, version, sources) + end + end + + # Installs the plugin from Definition object created by limited parsing of + # Gemfile searching for plugins to be installed + # + # @param [Definition] definition object + # @return [Hash] map of names to their specs they are installed with + def install_definition(definition) + def definition.lock(*); end + definition.resolve_remotely! + specs = definition.specs + + install_from_specs specs + end + + private + + def install_git(names, version, options) + uri = options.delete(:git) + options["uri"] = uri + + source_list = SourceList.new + source_list.add_git_source(options) + + # To support both sources + if options[:source] + source_list.add_rubygems_source("remotes" => options[:source]) + end + + deps = names.map {|name| Dependency.new name, version } + + definition = Definition.new(nil, deps, source_list, true) + install_definition(definition) + end + + # Installs the plugin from rubygems source and returns the path where the + # plugin was installed + # + # @param [String] name of the plugin gem to search in the source + # @param [Array] version of the gem to install + # @param [String, Array] source(s) to resolve the gem + # + # @return [Hash] map of names to the specs of plugins installed + def install_rubygems(names, version, sources) + deps = names.map {|name| Dependency.new name, version } + + source_list = SourceList.new + source_list.add_rubygems_source("remotes" => sources) + + definition = Definition.new(nil, deps, source_list, true) + install_definition(definition) + end + + # Installs the plugins and deps from the provided specs and returns map of + # gems to their paths + # + # @param specs to install + # + # @return [Hash] map of names to the specs + def install_from_specs(specs) + paths = {} + + specs.each do |spec| + spec.source.install spec + + paths[spec.name] = spec + end + + paths + end + end + end +end diff --git a/lib/bundler/plugin/installer/git.rb b/lib/bundler/plugin/installer/git.rb new file mode 100644 index 0000000000..fbb6c5e40e --- /dev/null +++ b/lib/bundler/plugin/installer/git.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + class Installer + class Git < Bundler::Source::Git + def cache_path + @cache_path ||= begin + git_scope = "#{base_name}-#{uri_hash}" + + Plugin.cache.join("bundler", "git", git_scope) + end + end + + def install_path + @install_path ||= begin + git_scope = "#{base_name}-#{shortref_for_path(revision)}" + + Plugin.root.join("bundler", "gems", git_scope) + end + end + + def version_message(spec) + "#{spec.name} #{spec.version}" + end + + def root + Plugin.root + end + + def generate_bin(spec, disable_extensions = false) + # Need to find a way without code duplication + # For now, we can ignore this + end + end + end + end +end diff --git a/lib/bundler/plugin/installer/rubygems.rb b/lib/bundler/plugin/installer/rubygems.rb new file mode 100644 index 0000000000..7ae74fa93b --- /dev/null +++ b/lib/bundler/plugin/installer/rubygems.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + class Installer + class Rubygems < Bundler::Source::Rubygems + def version_message(spec) + "#{spec.name} #{spec.version}" + end + + private + + def requires_sudo? + false # Will change on implementation of project level plugins + end + + def rubygems_dir + Plugin.root + end + + def cache_path + Plugin.cache + end + end + end + end +end diff --git a/lib/bundler/plugin/source_list.rb b/lib/bundler/plugin/source_list.rb new file mode 100644 index 0000000000..33f5e5afbd --- /dev/null +++ b/lib/bundler/plugin/source_list.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Bundler + # SourceList object to be used while parsing the Gemfile, setting the + # approptiate options to be used with Source classes for plugin installation + module Plugin + class SourceList < Bundler::SourceList + def initialize + @path_sources = [] + @git_sources = [] + @rubygems_aggregate = Plugin::Installer::Rubygems.new + @rubygems_sources = [] + end + + def add_git_source(options = {}) + add_source_to_list Plugin::Installer::Git.new(options), git_sources + end + + def add_rubygems_source(options = {}) + add_source_to_list Plugin::Installer::Rubygems.new(options), @rubygems_sources + end + + def all_sources + path_sources + git_sources + rubygems_sources + end + end + end +end -- cgit v1.2.3