* lib/rubygems: Update to RubyGems master 4bdc4f2. Important changes

  in this commit:

  RubyGems now chooses the test server port reliably.  Patch by akr.

  Partial implementation of bundler's Gemfile format.

  Refactorings to improve the new resolver.

  Fixes bugs in the resolver.

* test/rubygems:  Tests for the above.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@43643 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 604 insertions, 22 deletions

diff --git a/lib/rubygems/request_set/gem_dependency_api.rb b/lib/rubygems/request_set/gem_dependency_api.rb
index f11ffb12c3..e8f3138990 100644
--- a/lib/rubygems/request_set/gem_dependency_api.rb
+++ b/lib/rubygems/request_set/gem_dependency_api.rb
@@ -3,10 +3,114 @@
 
 class Gem::RequestSet::GemDependencyAPI
 
+  ENGINE_MAP = { # :nodoc:
+    :jruby        => %w[jruby],
+    :jruby_18     => %w[jruby],
+    :jruby_19     => %w[jruby],
+    :maglev       => %w[maglev],
+    :mri          => %w[ruby],
+    :mri_18       => %w[ruby],
+    :mri_19       => %w[ruby],
+    :mri_20       => %w[ruby],
+    :mri_21       => %w[ruby],
+    :rbx          => %w[rbx],
+    :ruby         => %w[ruby rbx maglev],
+    :ruby_18      => %w[ruby rbx maglev],
+    :ruby_19      => %w[ruby rbx maglev],
+    :ruby_20      => %w[ruby rbx maglev],
+    :ruby_21      => %w[ruby rbx maglev],
+  }
+
+  x86_mingw = Gem::Platform.new 'x86-mingw32'
+  x64_mingw = Gem::Platform.new 'x64-mingw32'
+
+  PLATFORM_MAP = { # :nodoc:
+    :jruby        => Gem::Platform::RUBY,
+    :jruby_18     => Gem::Platform::RUBY,
+    :jruby_19     => Gem::Platform::RUBY,
+    :maglev       => Gem::Platform::RUBY,
+    :mingw        => x86_mingw,
+    :mingw_18     => x86_mingw,
+    :mingw_19     => x86_mingw,
+    :mingw_20     => x86_mingw,
+    :mingw_21     => x86_mingw,
+    :mri          => Gem::Platform::RUBY,
+    :mri_18       => Gem::Platform::RUBY,
+    :mri_19       => Gem::Platform::RUBY,
+    :mri_20       => Gem::Platform::RUBY,
+    :mri_21       => Gem::Platform::RUBY,
+    :mswin        => Gem::Platform::RUBY,
+    :rbx          => Gem::Platform::RUBY,
+    :ruby         => Gem::Platform::RUBY,
+    :ruby_18      => Gem::Platform::RUBY,
+    :ruby_19      => Gem::Platform::RUBY,
+    :ruby_20      => Gem::Platform::RUBY,
+    :ruby_21      => Gem::Platform::RUBY,
+    :x64_mingw    => x64_mingw,
+    :x64_mingw_20 => x64_mingw,
+    :x64_mingw_21 => x64_mingw
+  }
+
+  gt_eq_0        = Gem::Requirement.new '>= 0'
+  tilde_gt_1_8_0 = Gem::Requirement.new '~> 1.8.0'
+  tilde_gt_1_9_0 = Gem::Requirement.new '~> 1.9.0'
+  tilde_gt_2_0_0 = Gem::Requirement.new '~> 2.0.0'
+  tilde_gt_2_1_0 = Gem::Requirement.new '~> 2.1.0'
+
+  VERSION_MAP = { # :nodoc:
+    :jruby        => gt_eq_0,
+    :jruby_18     => tilde_gt_1_8_0,
+    :jruby_19     => tilde_gt_1_9_0,
+    :maglev       => gt_eq_0,
+    :mingw        => gt_eq_0,
+    :mingw_18     => tilde_gt_1_8_0,
+    :mingw_19     => tilde_gt_1_9_0,
+    :mingw_20     => tilde_gt_2_0_0,
+    :mingw_21     => tilde_gt_2_1_0,
+    :mri          => gt_eq_0,
+    :mri_18       => tilde_gt_1_8_0,
+    :mri_19       => tilde_gt_1_9_0,
+    :mri_20       => tilde_gt_2_0_0,
+    :mri_21       => tilde_gt_2_1_0,
+    :mswin        => gt_eq_0,
+    :rbx          => gt_eq_0,
+    :ruby         => gt_eq_0,
+    :ruby_18      => tilde_gt_1_8_0,
+    :ruby_19      => tilde_gt_1_9_0,
+    :ruby_20      => tilde_gt_2_0_0,
+    :ruby_21      => tilde_gt_2_1_0,
+    :x64_mingw    => gt_eq_0,
+    :x64_mingw_20 => tilde_gt_2_0_0,
+    :x64_mingw_21 => tilde_gt_2_1_0,
+  }
+
+  WINDOWS = { # :nodoc:
+    :mingw        => :only,
+    :mingw_18     => :only,
+    :mingw_19     => :only,
+    :mingw_20     => :only,
+    :mingw_21     => :only,
+    :mri          => :never,
+    :mri_18       => :never,
+    :mri_19       => :never,
+    :mri_20       => :never,
+    :mri_21       => :never,
+    :mswin        => :only,
+    :rbx          => :never,
+    :ruby         => :never,
+    :ruby_18      => :never,
+    :ruby_19      => :never,
+    :ruby_20      => :never,
+    :ruby_21      => :never,
+    :x64_mingw    => :only,
+    :x64_mingw_20 => :only,
+    :x64_mingw_21 => :only,
+  }
+
   ##
-  # The dependency groups created by #group in the dependency API file.
+  # A Hash containing gem names and files to require from those gems.
 
-  attr_reader :dependency_groups
+  attr_reader :requires
 
   ##
   # A set of gems that are loaded via the +:path+ option to #gem
@@ -14,6 +118,11 @@ class Gem::RequestSet::GemDependencyAPI
   attr_reader :vendor_set # :nodoc:
 
   ##
+  # The groups of gems to exclude from installation
+
+  attr_accessor :without_groups
+
+  ##
   # Creates a new GemDependencyAPI that will add dependencies to the
   # Gem::RequestSet +set+ based on the dependency API description in +path+.
 
@@ -21,9 +130,13 @@ class Gem::RequestSet::GemDependencyAPI
     @set = set
     @path = path
 
-    @current_groups    = nil
-    @dependency_groups = Hash.new { |h, group| h[group] = [] }
-    @vendor_set        = @set.vendor_set
+    @current_groups   = nil
+    @current_platform = nil
+    @default_sources  = true
+    @requires         = Hash.new { |h, name| h[name] = [] }
+    @vendor_set       = @set.vendor_set
+    @gem_sources      = {}
+    @without_groups   = []
   end
 
   ##
@@ -47,10 +160,32 @@ class Gem::RequestSet::GemDependencyAPI
     options = requirements.pop if requirements.last.kind_of?(Hash)
     options ||= {}
 
-    if directory = options.delete(:path) then
-      @vendor_set.add_vendor_gem name, directory
+    source_set = gem_path name, options
+
+    return unless gem_platforms options
+
+    groups = gem_group name, options
+
+    return unless (groups & @without_groups).empty?
+
+    unless source_set then
+      raise ArgumentError,
+        "duplicate source (default) for gem #{name}" if
+          @gem_sources.include? name
+
+      @gem_sources[name] = :default
     end
 
+    gem_requires name, options
+
+    @set.gem name, *requirements
+  end
+
+  ##
+  # Handles the :group and :groups +options+ for the gem with the given
+  # +name+.
+
+  def gem_group name, options # :nodoc:
     g = options.delete :group
     all_groups  = g ? Array(g) : []
 
@@ -59,19 +194,81 @@ class Gem::RequestSet::GemDependencyAPI
 
     all_groups |= @current_groups if @current_groups
 
-    unless all_groups.empty? then
-      all_groups.each do |group|
-        gem_arguments = [name, *requirements]
-        gem_arguments << options unless options.empty?
-        @dependency_groups[group] << gem_arguments
+    all_groups
+  end
+
+  private :gem_group
+
+  ##
+  # Handles the path: option from +options+ for gem +name+.
+  #
+  # Returns +true+ if the path option was handled.
+
+  def gem_path name, options # :nodoc:
+    return unless directory = options.delete(:path)
+
+    raise ArgumentError,
+      "duplicate source path: #{directory} for gem #{name}" if
+        @gem_sources.include? name
+
+    @vendor_set.add_vendor_gem name, directory
+
+    @gem_sources[name] = directory
+
+    true
+  end
+
+  private :gem_path
+
+  ##
+  # Handles the platforms: option from +options+.  Returns true if the
+  # platform matches the current platform.
+
+  def gem_platforms options # :nodoc:
+    platform_names = Array(options.delete :platforms)
+    platform_names << @current_platform if @current_platform
+
+    return true if platform_names.empty?
+
+    platform_names.any? do |platform_name|
+      raise ArgumentError, "unknown platform #{platform_name.inspect}" unless
+        platform = PLATFORM_MAP[platform_name]
+
+      next false unless Gem::Platform.match platform
+
+      if engines = ENGINE_MAP[platform_name] then
+        next false unless engines.include? Gem.ruby_engine
+      end
+
+      case WINDOWS[platform_name]
+      when :only then
+        next false unless Gem.win_platform?
+      when :never then
+        next false if Gem.win_platform?
       end
 
-      return
+      VERSION_MAP[platform_name].satisfied_by? Gem.ruby_version
     end
+  end
 
-    @set.gem name, *requirements
+  private :gem_platforms
+
+  ##
+  # Handles the require: option from +options+ and adds those files, or the
+  # default file to the require list for +name+.
+
+  def gem_requires name, options # :nodoc:
+    if options.include? :require then
+      if requires = options.delete(:require) then
+        @requires[name].concat requires
+      end
+    else
+      @requires[name] << name
+    end
   end
 
+  private :gem_requires
+
   ##
   # Returns the basename of the file the dependencies were loaded from
 
@@ -96,9 +293,12 @@ class Gem::RequestSet::GemDependencyAPI
   # :category: Gem Dependencies DSL
 
   def platform what
-    if what == :ruby
-      yield
-    end
+    @current_platform = what
+
+    yield
+
+  ensure
+    @current_platform = nil
   end
 
   ##
@@ -112,23 +312,58 @@ class Gem::RequestSet::GemDependencyAPI
   # +:engine+ options from Bundler are currently ignored.
 
   def ruby version, options = {}
-    return true if version == RUBY_VERSION
+    engine         = options[:engine]
+    engine_version = options[:engine_version]
+
+    raise ArgumentError,
+          'you must specify engine_version along with the ruby engine' if
+            engine and not engine_version
+
+    unless RUBY_VERSION == version then
+      message = "Your Ruby version is #{RUBY_VERSION}, " +
+                "but your #{gem_deps_file} requires #{version}"
+
+      raise Gem::RubyVersionMismatch, message
+    end
+
+    if engine and engine != Gem.ruby_engine then
+      message = "Your ruby engine is #{Gem.ruby_engine}, " +
+                "but your #{gem_deps_file} requires #{engine}"
+
+      raise Gem::RubyVersionMismatch, message
+    end
 
-    message = "Your Ruby version is #{RUBY_VERSION}, " +
-              "but your #{gem_deps_file} specified #{version}"
+    if engine_version then
+      my_engine_version = Object.const_get "#{Gem.ruby_engine.upcase}_VERSION"
 
-    raise Gem::RubyVersionMismatch, message
+      if engine_version != my_engine_version then
+        message =
+          "Your ruby engine version is #{Gem.ruby_engine} #{my_engine_version}, " +
+          "but your #{gem_deps_file} requires #{engine} #{engine_version}"
+
+        raise Gem::RubyVersionMismatch, message
+      end
+    end
+
+    return true
   end
 
   ##
   # :category: Gem Dependencies DSL
+  #
+  # Sets +url+ as a source for gems for this dependency API.
 
   def source url
+    Gem.sources.clear if @default_sources
+
+    @default_sources = false
+
+    Gem.sources << url
   end
 
   # TODO: remove this typo name at RubyGems 3.0
 
-  Gem::RequestSet::DepedencyAPI = self # :nodoc:
+  Gem::RequestSet::GemDepedencyAPI = self # :nodoc:
 
 end
 
diff --git a/lib/rubygems/request_set/lockfile.rb b/lib/rubygems/request_set/lockfile.rb
new file mode 100644
index 0000000000..a9c419549d
--- /dev/null
+++ b/lib/rubygems/request_set/lockfile.rb
@@ -0,0 +1,347 @@
+require 'pathname'
+
+class Gem::RequestSet::Lockfile
+
+  ##
+  # Raised when a lockfile cannot be parsed
+
+  class ParseError < Gem::Exception
+
+    ##
+    # The column where the error was encountered
+
+    attr_reader :column
+
+    ##
+    # The line where the error was encountered
+
+    attr_reader :line
+
+    ##
+    # The location of the lock file
+
+    attr_reader :path
+
+    ##
+    # Raises a ParseError with the given +message+ which was encountered at a
+    # +line+ and +column+ while parsing.
+
+    def initialize message, line, column, path
+      @line   = line
+      @column = column
+      @path   = path
+      super "#{message} (at #{line}:#{column})"
+    end
+
+  end
+
+  ##
+  # The platforms for this Lockfile
+
+  attr_reader :platforms
+
+  ##
+  # Creates a new Lockfile for the given +request_set+ and +gem_deps_file+
+  # location.
+
+  def initialize request_set, gem_deps_file
+    @set           = request_set
+    @gem_deps_file = Pathname(gem_deps_file).expand_path
+    @gem_deps_dir  = @gem_deps_file.dirname
+
+    @current_token  = nil
+    @line           = 0
+    @line_pos       = 0
+    @platforms      = []
+    @tokens         = []
+  end
+
+  def add_DEPENDENCIES out # :nodoc:
+    out << "DEPENDENCIES"
+
+    @set.dependencies.sort.map do |dependency|
+      source = @requests.find do |req|
+        req.name == dependency.name and
+          req.spec.class == Gem::DependencyResolver::VendorSpecification
+      end
+
+      source_dep = '!' if source
+
+      requirement = dependency.requirement
+
+      out << "  #{dependency.name}#{source_dep}#{requirement.for_lockfile}"
+    end
+
+    out << nil
+  end
+
+  def add_GEM out # :nodoc:
+    out << "GEM"
+
+    source_groups = @spec_groups.values.flatten.group_by do |request|
+      request.spec.source.uri
+    end
+
+    source_groups.map do |group, requests|
+      out << "  remote: #{group}"
+      out << "  specs:"
+
+      requests.sort_by { |request| request.name }.each do |request|
+        platform = "-#{request.spec.platform}" unless
+          Gem::Platform::RUBY == request.spec.platform
+
+        out << "    #{request.name} (#{request.version}#{platform})"
+
+        request.full_spec.dependencies.sort.each do |dependency|
+          requirement = dependency.requirement
+          out << "      #{dependency.name}#{requirement.for_lockfile}"
+        end
+      end
+    end
+
+    out << nil
+  end
+
+  def add_PATH out # :nodoc:
+    return unless path_requests =
+      @spec_groups.delete(Gem::DependencyResolver::VendorSpecification)
+
+    out << "PATH"
+    path_requests.each do |request|
+      directory = Pathname(request.spec.source.uri).expand_path
+
+      out << "  remote: #{directory.relative_path_from @gem_deps_dir}"
+      out << "  specs:"
+      out << "    #{request.name} (#{request.version})"
+    end
+
+    out << nil
+  end
+
+  def add_PLATFORMS out # :nodoc:
+    out << "PLATFORMS"
+
+    platforms = @requests.map { |request| request.spec.platform }.uniq
+    platforms.delete Gem::Platform::RUBY if platforms.length > 1
+
+    platforms.each do |platform|
+      out << "  #{platform}"
+    end
+
+    out << nil
+  end
+
+  ##
+  # Gets the next token for a Lockfile
+
+  def get expected_type = nil, expected_value = nil # :nodoc:
+    @current_token = @tokens.shift
+
+    type, value, line, column = @current_token
+
+    if expected_type and expected_type != type then
+      unget
+
+      message = "unexpected token [#{type.inspect}, #{value.inspect}], " +
+                "expected #{expected_type.inspect}"
+
+      raise ParseError.new message, line, column, "#{@gem_deps_file}.lock"
+    end
+
+    if expected_value and expected_value != value then
+      unget
+
+      message = "unexpected token [#{type.inspect}, #{value.inspect}], " +
+                "expected [#{expected_type.inspect}, #{expected_value.inspect}]"
+
+      raise ParseError.new message, line, column, "#{@gem_deps_file}.lock"
+    end
+
+    @current_token
+  end
+
+  def parse # :nodoc:
+    tokenize
+
+    until @tokens.empty? do
+      type, data, column, line = get
+
+      case type
+      when :section then
+        skip :newline
+
+        case data
+        when 'DEPENDENCIES' then
+          parse_DEPENDENCIES
+        when 'GEM' then
+          parse_GEM
+        when 'PLATFORMS' then
+          parse_PLATFORMS
+        else
+          type, = get until @tokens.empty? or peek.first == :section
+        end
+      else
+        raise "BUG: unhandled token #{type} (#{data.inspect}) at #{line}:#{column}"
+      end
+    end
+  end
+
+  def parse_DEPENDENCIES # :nodoc:
+    while not @tokens.empty? and :text == peek.first do
+      _, name, = get :text
+
+      @set.gem name
+
+      skip :newline
+    end
+  end
+
+  def parse_GEM # :nodoc:
+    get :entry, 'remote'
+    _, data, = get :text
+
+    source = Gem::Source.new data
+
+    skip :newline
+
+    get :entry, 'specs'
+
+    skip :newline
+
+    set = Gem::DependencyResolver::LockSet.new source
+
+    while not @tokens.empty? and :text == peek.first do
+      _, name, = get :text
+
+      case peek[0]
+      when :newline then # ignore
+      when :l_paren then
+        get :l_paren
+
+        _, version, = get :text
+
+        get :r_paren
+
+        set.add name, version, Gem::Platform::RUBY
+      else
+        raise "BUG: unknown token #{peek}"
+      end
+
+      skip :newline
+    end
+
+    @set.sets << set
+  end
+
+  def parse_PLATFORMS # :nodoc:
+    while not @tokens.empty? and :text == peek.first do
+      _, name, = get :text
+
+      @platforms << name
+
+      skip :newline
+    end
+  end
+
+  ##
+  # Peeks at the next token for Lockfile
+
+  def peek # :nodoc:
+    @tokens.first
+  end
+
+  def skip type # :nodoc:
+    get while not @tokens.empty? and peek.first == type
+  end
+
+  def to_s
+    @set.resolve
+
+    out = []
+
+    @requests = @set.sorted_requests
+
+    @spec_groups = @requests.group_by do |request|
+      request.spec.class
+    end
+
+    add_PATH out
+
+    add_GEM out
+
+    add_PLATFORMS out
+
+    add_DEPENDENCIES out
+
+    out.join "\n"
+  end
+
+  ##
+  # Calculates the column (by byte) and the line of the current token based on
+  # +byte_offset+.
+
+  def token_pos byte_offset # :nodoc:
+    [byte_offset - @line_pos, @line]
+  end
+
+  def tokenize # :nodoc:
+    @line     = 0
+    @line_pos = 0
+
+    @platforms     = []
+    @tokens        = []
+    @current_token = nil
+
+    lock_file = "#{@gem_deps_file}.lock"
+
+    @input = File.read lock_file
+    s      = StringScanner.new @input
+
+    until s.eos? do
+      pos = s.pos
+
+      # leading whitespace is for the user's convenience
+      next if s.scan(/ +/)
+
+      if s.scan(/[<|=>]{7}/) then
+        message = "your #{lock_file} contains merge conflict markers"
+        line, column = token_pos pos
+
+        raise ParseError.new message, line, column, lock_file
+      end
+
+      @tokens <<
+        case
+        when s.scan(/\r?\n/) then
+          token = [:newline, nil, *token_pos(pos)]
+          @line_pos = s.pos
+          @line += 1
+          token
+        when s.scan(/[A-Z]+/) then
+          [:section, s.matched, *token_pos(pos)]
+        when s.scan(/([a-z]+):\s/) then
+          s.pos -= 1 # rewind for possible newline
+          [:entry, s[1], *token_pos(pos)]
+        when s.scan(/\(/) then
+          [:l_paren, nil, *token_pos(pos)]
+        when s.scan(/\)/) then
+          [:r_paren, nil, *token_pos(pos)]
+        when s.scan(/[^\s)]*/) then
+          [:text, s.matched, *token_pos(pos)]
+        else
+          raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
+        end
+    end
+
+    @tokens
+  end
+
+  ##
+  # Ungets the last token retrieved by #get
+
+  def unget # :nodoc:
+    @tokens.unshift @current_token
+  end
+
+end
+