[rubygems/rubygems] Extract generate_index command to rubygems-generate_index gem

So generate_index can be implemented with dependencies, such as the compact index

Took this approach from feedback in https://github.com/rubygems/rubygems/pull/6853

Running `gem generate_index` by default will use an installed rubygems-generate_index, or install and then use the command from the gem

Apply suggestions from code review

https://github.com/rubygems/rubygems/commit/fc1cb9bc9e

Co-authored-by: Hiroshi SHIBATA <hsbt@ruby-lang.org>

4 files changed, 41 insertions, 504 deletions

diff --git a/lib/rubygems/command_manager.rb b/lib/rubygems/command_manager.rb
index c53f0231af..1028283ffa 100644
--- a/lib/rubygems/command_manager.rb
+++ b/lib/rubygems/command_manager.rb
@@ -249,6 +249,7 @@ class Gem::CommandManager
   def invoke_command(args, build_args)
     cmd_name = args.shift.downcase
     cmd = find_command cmd_name
+    terminate_interaction 1 unless cmd
     cmd.deprecation_warning if cmd.deprecated?
     cmd.invoke_with_build_args args, build_args
   end
diff --git a/lib/rubygems/commands/generate_index_command.rb b/lib/rubygems/commands/generate_index_command.rb
index 0bf74355ae..13be92593b 100644
--- a/lib/rubygems/commands/generate_index_command.rb
+++ b/lib/rubygems/commands/generate_index_command.rb
@@ -1,86 +1,51 @@
 # frozen_string_literal: true
 
 require_relative "../command"
-require_relative "../indexer"
 
-##
-# Generates a index files for use as a gem server.
-#
-# See `gem help generate_index`
-
-class Gem::Commands::GenerateIndexCommand < Gem::Command
-  def initialize
-    super "generate_index",
-          "Generates the index files for a gem server directory",
-          directory: ".", build_modern: true
+unless defined? Gem::Commands::GenerateIndexCommand
+  class Gem::Commands::GenerateIndexCommand < Gem::Command
+    module RubygemsTrampoline
+      def description # :nodoc:
+        <<~EOF
+          The generate_index command has been moved to the rubygems-generate_index gem.
+        EOF
+      end
 
-    add_option "-d", "--directory=DIRNAME",
-               "repository base dir containing gems subdir" do |dir, options|
-      options[:directory] = File.expand_path dir
-    end
+      def execute
+        alert_error "Install the rubygems-generate_index gem for the generate_index command"
+      end
 
-    add_option "--[no-]modern",
-               "Generate indexes for RubyGems",
-               "(always true)" do |value, options|
-      options[:build_modern] = value
+      def invoke_with_build_args(args, build_args)
+        name = "rubygems-generate_index"
+        spec = begin
+          Gem::Specification.find_by_name(name)
+        rescue Gem::LoadError
+          require "rubygems/dependency_installer"
+          Gem.install(name, Gem::Requirement.default, Gem::DependencyInstaller::DEFAULT_OPTIONS).find {|s| s.name == name }
+        end
+
+        # remove the methods defined in this file so that the methods defined in the gem are used instead,
+        # and without a method redefinition warning
+        %w[description execute invoke_with_build_args].each do |method|
+          RubygemsTrampoline.remove_method(method)
+        end
+        self.class.singleton_class.remove_method(:new)
+
+        spec.activate
+        Gem.load_plugin_files spec.matches_for_glob("rubygems_plugin#{Gem.suffix_pattern}")
+
+        self.class.new.invoke_with_build_args(args, build_args)
+      end
     end
+    private_constant :RubygemsTrampoline
 
-    deprecate_option("--modern", version: "4.0", extra_msg: "Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.")
-    deprecate_option("--no-modern", version: "4.0", extra_msg: "The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.")
-
-    add_option "--update",
-               "Update modern indexes with gems added",
-               "since the last update" do |value, options|
-      options[:update] = value
+    # remove_method(:initialize) warns, but removing new does not warn
+    def self.new
+      command = allocate
+      command.send(:initialize, "generate_index", "Generates the index files for a gem server directory (requires rubygems-generate_index)")
+      command
     end
-  end
-
-  def defaults_str # :nodoc:
-    "--directory . --modern"
-  end
-
-  def description # :nodoc:
-    <<-EOF
-The generate_index command creates a set of indexes for serving gems
-statically.  The command expects a 'gems' directory under the path given to
-the --directory option.  The given directory will be the directory you serve
-as the gem repository.
 
-For `gem generate_index --directory /path/to/repo`, expose /path/to/repo via
-your HTTP server configuration (not /path/to/repo/gems).
-
-When done, it will generate a set of files like this:
-
-  gems/*.gem                                   # .gem files you want to
-                                               # index
-
-  specs.<version>.gz                           # specs index
-  latest_specs.<version>.gz                    # latest specs index
-  prerelease_specs.<version>.gz                # prerelease specs index
-  quick/Marshal.<version>/<gemname>.gemspec.rz # Marshal quick index file
-
-The .rz extension files are compressed with the inflate algorithm.
-The Marshal version number comes from ruby's Marshal::MAJOR_VERSION and
-Marshal::MINOR_VERSION constants.  It is used to ensure compatibility.
-    EOF
-  end
-
-  def execute
-    # This is always true because it's the only way now.
-    options[:build_modern] = true
-
-    if !File.exist?(options[:directory]) ||
-       !File.directory?(options[:directory])
-      alert_error "unknown directory name #{options[:directory]}."
-      terminate_interaction 1
-    else
-      indexer = Gem::Indexer.new options.delete(:directory), options
-
-      if options[:update]
-        indexer.update_index
-      else
-        indexer.generate_index
-      end
-    end
+    prepend(RubygemsTrampoline)
   end
 end
diff --git a/lib/rubygems/commands/help_command.rb b/lib/rubygems/commands/help_command.rb
index 043e7d3691..8994f1aa09 100644
--- a/lib/rubygems/commands/help_command.rb
+++ b/lib/rubygems/commands/help_command.rb
@@ -333,7 +333,7 @@ platform.
     @command_manager.command_names.each do |cmd_name|
       command = @command_manager[cmd_name]
 
-      next if command.deprecated?
+      next if command&.deprecated?
 
       summary =
         if command
diff --git a/lib/rubygems/indexer.rb b/lib/rubygems/indexer.rb
deleted file mode 100644
index 82c334672f..0000000000
--- a/lib/rubygems/indexer.rb
+++ /dev/null
@@ -1,429 +0,0 @@
-# frozen_string_literal: true
-
-require_relative "../rubygems"
-require_relative "package"
-require "tmpdir"
-
-##
-# Top level class for building the gem repository index.
-
-class Gem::Indexer
-  include Gem::UserInteraction
-
-  ##
-  # Build indexes for RubyGems 1.2.0 and newer when true
-
-  attr_accessor :build_modern
-
-  ##
-  # Index install location
-
-  attr_reader :dest_directory
-
-  ##
-  # Specs index install location
-
-  attr_reader :dest_specs_index
-
-  ##
-  # Latest specs index install location
-
-  attr_reader :dest_latest_specs_index
-
-  ##
-  # Prerelease specs index install location
-
-  attr_reader :dest_prerelease_specs_index
-
-  ##
-  # Index build directory
-
-  attr_reader :directory
-
-  ##
-  # Create an indexer that will index the gems in +directory+.
-
-  def initialize(directory, options = {})
-    require "fileutils"
-    require "tmpdir"
-    require "zlib"
-
-    options = { build_modern: true }.merge options
-
-    @build_modern = options[:build_modern]
-
-    @dest_directory = directory
-    @directory = Dir.mktmpdir "gem_generate_index"
-
-    marshal_name = "Marshal.#{Gem.marshal_version}"
-
-    @master_index = File.join @directory, "yaml"
-    @marshal_index = File.join @directory, marshal_name
-
-    @quick_dir = File.join @directory, "quick"
-    @quick_marshal_dir = File.join @quick_dir, marshal_name
-    @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH
-
-    @quick_index = File.join @quick_dir, "index"
-    @latest_index = File.join @quick_dir, "latest_index"
-
-    @specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
-    @latest_specs_index =
-      File.join(@directory, "latest_specs.#{Gem.marshal_version}")
-    @prerelease_specs_index =
-      File.join(@directory, "prerelease_specs.#{Gem.marshal_version}")
-    @dest_specs_index =
-      File.join(@dest_directory, "specs.#{Gem.marshal_version}")
-    @dest_latest_specs_index =
-      File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}")
-    @dest_prerelease_specs_index =
-      File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}")
-
-    @files = []
-  end
-
-  ##
-  # Build various indices
-
-  def build_indices
-    specs = map_gems_to_specs gem_file_list
-    Gem::Specification._resort! specs
-    build_marshal_gemspecs specs
-    build_modern_indices specs if @build_modern
-
-    compress_indices
-  end
-
-  ##
-  # Builds Marshal quick index gemspecs.
-
-  def build_marshal_gemspecs(specs)
-    count = specs.count
-    progress = ui.progress_reporter count,
-                                    "Generating Marshal quick index gemspecs for #{count} gems",
-                                    "Complete"
-
-    files = []
-
-    Gem.time "Generated Marshal quick index gemspecs" do
-      specs.each do |spec|
-        next if spec.default_gem?
-        spec_file_name = "#{spec.original_name}.gemspec.rz"
-        marshal_name = File.join @quick_marshal_dir, spec_file_name
-
-        marshal_zipped = Gem.deflate Marshal.dump(spec)
-
-        File.open marshal_name, "wb" do |io|
-          io.write marshal_zipped
-        end
-
-        files << marshal_name
-
-        progress.updated spec.original_name
-      end
-
-      progress.done
-    end
-
-    @files << @quick_marshal_dir
-
-    files
-  end
-
-  ##
-  # Build a single index for RubyGems 1.2 and newer
-
-  def build_modern_index(index, file, name)
-    say "Generating #{name} index"
-
-    Gem.time "Generated #{name} index" do
-      File.open(file, "wb") do |io|
-        specs = index.map do |*spec|
-          # We have to splat here because latest_specs is an array, while the
-          # others are hashes.
-          spec = spec.flatten.last
-          platform = spec.original_platform
-
-          # win32-api-1.0.4-x86-mswin32-60
-          unless String === platform
-            alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
-            next
-          end
-
-          platform = Gem::Platform::RUBY if platform.nil? || platform.empty?
-          [spec.name, spec.version, platform]
-        end
-
-        specs = compact_specs(specs)
-        Marshal.dump(specs, io)
-      end
-    end
-  end
-
-  ##
-  # Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease
-
-  def build_modern_indices(specs)
-    prerelease, released = specs.partition do |s|
-      s.version.prerelease?
-    end
-    latest_specs =
-      Gem::Specification._latest_specs specs
-
-    build_modern_index(released.sort, @specs_index, "specs")
-    build_modern_index(latest_specs.sort, @latest_specs_index, "latest specs")
-    build_modern_index(prerelease.sort, @prerelease_specs_index,
-                       "prerelease specs")
-
-    @files += [@specs_index,
-               "#{@specs_index}.gz",
-               @latest_specs_index,
-               "#{@latest_specs_index}.gz",
-               @prerelease_specs_index,
-               "#{@prerelease_specs_index}.gz"]
-  end
-
-  def map_gems_to_specs(gems)
-    gems.map do |gemfile|
-      if File.size(gemfile) == 0
-        alert_warning "Skipping zero-length gem: #{gemfile}"
-        next
-      end
-
-      begin
-        spec = Gem::Package.new(gemfile).spec
-        spec.loaded_from = gemfile
-
-        spec.abbreviate
-        spec.sanitize
-
-        spec
-      rescue SignalException
-        alert_error "Received signal, exiting"
-        raise
-      rescue StandardError => e
-        msg = ["Unable to process #{gemfile}",
-               "#{e.message} (#{e.class})",
-               "\t#{e.backtrace.join "\n\t"}"].join("\n")
-        alert_error msg
-      end
-    end.compact
-  end
-
-  ##
-  # Compresses indices on disk
-  #--
-  # All future files should be compressed using gzip, not deflate
-
-  def compress_indices
-    say "Compressing indices"
-
-    Gem.time "Compressed indices" do
-      if @build_modern
-        gzip @specs_index
-        gzip @latest_specs_index
-        gzip @prerelease_specs_index
-      end
-    end
-  end
-
-  ##
-  # Compacts Marshal output for the specs index data source by using identical
-  # objects as much as possible.
-
-  def compact_specs(specs)
-    names = {}
-    versions = {}
-    platforms = {}
-
-    specs.map do |(name, version, platform)|
-      names[name] = name unless names.include? name
-      versions[version] = version unless versions.include? version
-      platforms[platform] = platform unless platforms.include? platform
-
-      [names[name], versions[version], platforms[platform]]
-    end
-  end
-
-  ##
-  # Compress +filename+ with +extension+.
-
-  def compress(filename, extension)
-    data = Gem.read_binary filename
-
-    zipped = Gem.deflate data
-
-    File.open "#{filename}.#{extension}", "wb" do |io|
-      io.write zipped
-    end
-  end
-
-  ##
-  # List of gem file names to index.
-
-  def gem_file_list
-    Gem::Util.glob_files_in_dir("*.gem", File.join(@dest_directory, "gems"))
-  end
-
-  ##
-  # Builds and installs indices.
-
-  def generate_index
-    make_temp_directories
-    build_indices
-    install_indices
-  rescue SignalException
-  ensure
-    FileUtils.rm_rf @directory
-  end
-
-  ##
-  # Zlib::GzipWriter wrapper that gzips +filename+ on disk.
-
-  def gzip(filename)
-    Zlib::GzipWriter.open "#{filename}.gz" do |io|
-      io.write Gem.read_binary(filename)
-    end
-  end
-
-  ##
-  # Install generated indices into the destination directory.
-
-  def install_indices
-    verbose = Gem.configuration.really_verbose
-
-    say "Moving index into production dir #{@dest_directory}" if verbose
-
-    files = @files
-    files.delete @quick_marshal_dir if files.include? @quick_dir
-
-    if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir)
-      files.delete @quick_marshal_dir
-
-      dst_name = File.join(@dest_directory, @quick_marshal_dir_base)
-
-      FileUtils.mkdir_p File.dirname(dst_name), verbose: verbose
-      FileUtils.rm_rf dst_name, verbose: verbose
-      FileUtils.mv(@quick_marshal_dir, dst_name,
-                   verbose: verbose, force: true)
-    end
-
-    files = files.map do |path|
-      path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK?
-    end
-
-    files.each do |file|
-      src_name = File.join @directory, file
-      dst_name = File.join @dest_directory, file
-
-      FileUtils.rm_rf dst_name, verbose: verbose
-      FileUtils.mv(src_name, @dest_directory,
-                   verbose: verbose, force: true)
-    end
-  end
-
-  ##
-  # Make directories for index generation
-
-  def make_temp_directories
-    FileUtils.rm_rf @directory
-    FileUtils.mkdir_p @directory, mode: 0o700
-    FileUtils.mkdir_p @quick_marshal_dir
-  end
-
-  ##
-  # Ensure +path+ and path with +extension+ are identical.
-
-  def paranoid(path, extension)
-    data = Gem.read_binary path
-    compressed_data = Gem.read_binary "#{path}.#{extension}"
-
-    unless data == Gem::Util.inflate(compressed_data)
-      raise "Compressed file #{compressed_path} does not match uncompressed file #{path}"
-    end
-  end
-
-  ##
-  # Perform an in-place update of the repository from newly added gems.
-
-  def update_index
-    make_temp_directories
-
-    specs_mtime = File.stat(@dest_specs_index).mtime
-    newest_mtime = Time.at 0
-
-    updated_gems = gem_file_list.select do |gem|
-      gem_mtime = File.stat(gem).mtime
-      newest_mtime = gem_mtime if gem_mtime > newest_mtime
-      gem_mtime >= specs_mtime
-    end
-
-    if updated_gems.empty?
-      say "No new gems"
-      terminate_interaction 0
-    end
-
-    specs = map_gems_to_specs updated_gems
-    prerelease, released = specs.partition {|s| s.version.prerelease? }
-
-    files = build_marshal_gemspecs specs
-
-    Gem.time "Updated indexes" do
-      update_specs_index released, @dest_specs_index, @specs_index
-      update_specs_index released, @dest_latest_specs_index, @latest_specs_index
-      update_specs_index(prerelease,
-                         @dest_prerelease_specs_index,
-                         @prerelease_specs_index)
-    end
-
-    compress_indices
-
-    verbose = Gem.configuration.really_verbose
-
-    say "Updating production dir #{@dest_directory}" if verbose
-
-    files << @specs_index
-    files << "#{@specs_index}.gz"
-    files << @latest_specs_index
-    files << "#{@latest_specs_index}.gz"
-    files << @prerelease_specs_index
-    files << "#{@prerelease_specs_index}.gz"
-
-    files = files.map do |path|
-      path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK?
-    end
-
-    files.each do |file|
-      src_name = File.join @directory, file
-      dst_name = File.join @dest_directory, file # REFACTOR: duped above
-
-      FileUtils.mv src_name, dst_name, verbose: verbose,
-                                       force: true
-
-      File.utime newest_mtime, newest_mtime, dst_name
-    end
-  ensure
-    FileUtils.rm_rf @directory
-  end
-
-  ##
-  # Combines specs in +index+ and +source+ then writes out a new copy to
-  # +dest+.  For a latest index, does not ensure the new file is minimal.
-
-  def update_specs_index(index, source, dest)
-    Gem.load_safe_marshal
-    specs_index = Gem::SafeMarshal.safe_load Gem.read_binary(source)
-
-    index.each do |spec|
-      platform = spec.original_platform
-      platform = Gem::Platform::RUBY if platform.nil? || platform.empty?
-      specs_index << [spec.name, spec.version, platform]
-    end
-
-    specs_index = compact_specs specs_index.uniq.sort
-
-    File.open dest, "wb" do |io|
-      Marshal.dump specs_index, io
-    end
-  end
-end