aboutsummaryrefslogtreecommitdiffstats
path: root/lib/rubygems/indexer.rb
diff options
context:
space:
mode:
Diffstat (limited to 'lib/rubygems/indexer.rb')
-rw-r--r--lib/rubygems/indexer.rb560
1 files changed, 451 insertions, 109 deletions
diff --git a/lib/rubygems/indexer.rb b/lib/rubygems/indexer.rb
index e2dd57d3fe..5ebde1b31c 100644
--- a/lib/rubygems/indexer.rb
+++ b/lib/rubygems/indexer.rb
@@ -6,6 +6,7 @@ require 'rubygems'
require 'rubygems/format'
begin
+ gem 'builder'
require 'builder/xchar'
rescue LoadError
end
@@ -18,11 +19,36 @@ class Gem::Indexer
include Gem::UserInteraction
##
+ # Build indexes for RubyGems older than 1.2.0 when true
+
+ attr_accessor :build_legacy
+
+ ##
+ # Build indexes for RubyGems 1.2.0 and newer when true
+
+ attr_accessor :build_modern
+
+ ##
# Index install location
attr_reader :dest_directory
##
+ # Specs index install location
+
+ attr_reader :dest_specs_index
+
+ ##
+ # Latest specs index install location
+
+ attr_reader :dest_latest_specs_index
+
+ ##
+ # Prerelease specs index install location
+
+ attr_reader :dest_prerelease_specs_index
+
+ ##
# Index build directory
attr_reader :directory
@@ -30,12 +56,21 @@ class Gem::Indexer
##
# Create an indexer that will index the gems in +directory+.
- def initialize(directory)
+ def initialize(directory, options = {})
unless ''.respond_to? :to_xs then
fail "Gem::Indexer requires that the XML Builder library be installed:" \
"\n\tgem install builder"
end
+ options = { :build_legacy => true, :build_modern => true }.merge options
+
+ @build_legacy = options[:build_legacy]
+ @build_modern = options[:build_modern]
+
+ @rss_title = options[:rss_title]
+ @rss_host = options[:rss_host]
+ @rss_gems_host = options[:rss_gems_host]
+
@dest_directory = directory
@directory = File.join Dir.tmpdir, "gem_generate_index_#{$$}"
@@ -54,22 +89,19 @@ class Gem::Indexer
@specs_index = File.join @directory, "specs.#{Gem.marshal_version}"
@latest_specs_index = File.join @directory,
"latest_specs.#{Gem.marshal_version}"
+ @prerelease_specs_index = File.join(@directory,
+ "prerelease_specs.#{Gem.marshal_version}")
- files = [
- @specs_index,
- "#{@specs_index}.gz",
- @latest_specs_index,
- "#{@latest_specs_index}.gz",
- @quick_dir,
- @master_index,
- "#{@master_index}.Z",
- @marshal_index,
- "#{@marshal_index}.Z",
- ]
-
- @files = files.map do |path|
- path.sub @directory, ''
- end
+ @dest_specs_index = File.join @dest_directory,
+ "specs.#{Gem.marshal_version}"
+ @dest_latest_specs_index = File.join @dest_directory,
+ "latest_specs.#{Gem.marshal_version}"
+ @dest_prerelease_specs_index = File.join @dest_directory,
+ "prerelease_specs.#{Gem.marshal_version}"
+
+ @rss_index = File.join @directory, 'index.rss'
+
+ @files = []
end
##
@@ -91,159 +123,367 @@ class Gem::Indexer
# Build various indicies
def build_indicies(index)
+ # Marshal gemspecs are used by both modern and legacy RubyGems
+ build_marshal_gemspecs index
+ build_legacy_indicies index if @build_legacy
+ build_modern_indicies index if @build_modern
+ build_rss index
+
+ compress_indicies
+ end
+
+ ##
+ # Builds indicies for RubyGems older than 1.2.x
+
+ def build_legacy_indicies(index)
progress = ui.progress_reporter index.size,
- "Generating quick index gemspecs for #{index.size} gems",
+ "Generating YAML quick index gemspecs for #{index.size} gems",
"Complete"
- index.each do |original_name, spec|
- spec_file_name = "#{original_name}.gemspec.rz"
- yaml_name = File.join @quick_dir, spec_file_name
- marshal_name = File.join @quick_marshal_dir, spec_file_name
+ Gem.time 'Generated YAML quick index gemspecs' do
+ index.each do |original_name, spec|
+ spec_file_name = "#{original_name}.gemspec.rz"
+ yaml_name = File.join @quick_dir, spec_file_name
- yaml_zipped = Gem.deflate spec.to_yaml
- open yaml_name, 'wb' do |io| io.write yaml_zipped end
+ yaml_zipped = Gem.deflate spec.to_yaml
+ open yaml_name, 'wb' do |io| io.write yaml_zipped end
- marshal_zipped = Gem.deflate Marshal.dump(spec)
- open marshal_name, 'wb' do |io| io.write marshal_zipped end
+ progress.updated original_name
+ end
- progress.updated original_name
+ progress.done
end
- progress.done
-
- say "Generating specs index"
+ say "Generating quick index"
- open @specs_index, 'wb' do |io|
- specs = index.sort.map do |_, spec|
- platform = spec.original_platform
- platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
- [spec.name, spec.version, platform]
+ Gem.time 'Generated quick index' do
+ open @quick_index, 'wb' do |io|
+ io.puts index.sort.map { |_, spec| spec.original_name }
end
-
- specs = compact_specs specs
-
- Marshal.dump specs, io
end
- say "Generating latest specs index"
+ say "Generating latest index"
- open @latest_specs_index, 'wb' do |io|
- specs = index.latest_specs.sort.map do |spec|
- platform = spec.original_platform
- platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
- [spec.name, spec.version, platform]
+ Gem.time 'Generated latest index' do
+ open @latest_index, 'wb' do |io|
+ io.puts index.latest_specs.sort.map { |spec| spec.original_name }
end
+ end
- specs = compact_specs specs
+ # Don't need prerelease legacy index
+
+ say "Generating Marshal master index"
- Marshal.dump specs, io
+ Gem.time 'Generated Marshal master index' do
+ open @marshal_index, 'wb' do |io|
+ io.write index.dump
+ end
end
- say "Generating quick index"
+ progress = ui.progress_reporter index.size,
+ "Generating YAML master index for #{index.size} gems (this may take a while)",
+ "Complete"
- quick_index = File.join @quick_dir, 'index'
- open quick_index, 'wb' do |io|
- io.puts index.sort.map { |_, spec| spec.original_name }
- end
+ Gem.time 'Generated YAML master index' do
+ open @master_index, 'wb' do |io|
+ io.puts "--- !ruby/object:#{index.class}"
+ io.puts "gems:"
- say "Generating latest index"
+ gems = index.sort_by { |name, gemspec| gemspec.sort_obj }
+ gems.each do |original_name, gemspec|
+ yaml = gemspec.to_yaml.gsub(/^/, ' ')
+ yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here
+ io.print " #{original_name}:"
+ io.puts yaml
+
+ progress.updated original_name
+ end
+ end
- latest_index = File.join @quick_dir, 'latest_index'
- open latest_index, 'wb' do |io|
- io.puts index.latest_specs.sort.map { |spec| spec.original_name }
+ progress.done
end
- say "Generating Marshal master index"
+ @files << @quick_dir
+ @files << @master_index
+ @files << "#{@master_index}.Z"
+ @files << @marshal_index
+ @files << "#{@marshal_index}.Z"
+ end
- open @marshal_index, 'wb' do |io|
- io.write index.dump
- end
+ ##
+ # Builds Marshal quick index gemspecs.
+ def build_marshal_gemspecs(index)
progress = ui.progress_reporter index.size,
- "Generating YAML master index for #{index.size} gems (this may take a while)",
+ "Generating Marshal quick index gemspecs for #{index.size} gems",
"Complete"
- open @master_index, 'wb' do |io|
- io.puts "--- !ruby/object:#{index.class}"
- io.puts "gems:"
+ files = []
- gems = index.sort_by { |name, gemspec| gemspec.sort_obj }
- gems.each do |original_name, gemspec|
- yaml = gemspec.to_yaml.gsub(/^/, ' ')
- yaml = yaml.sub(/\A ---/, '') # there's a needed extra ' ' here
- io.print " #{original_name}:"
- io.puts yaml
+ Gem.time 'Generated Marshal quick index gemspecs' do
+ (index.gems.merge(index.prerelease_gems)).each do |original_name, spec|
+ spec_file_name = "#{original_name}.gemspec.rz"
+ marshal_name = File.join @quick_marshal_dir, spec_file_name
+
+ marshal_zipped = Gem.deflate Marshal.dump(spec)
+ open marshal_name, 'wb' do |io| io.write marshal_zipped end
+
+ files << marshal_name
progress.updated original_name
end
+
+ progress.done
end
- progress.done
+ @files << @quick_marshal_dir
- say "Compressing indicies"
- # use gzip for future files.
+ files
+ end
+
+ ##
+ # Build a single index for RubyGems 1.2 and newer
+
+ def build_modern_index(index, file, name)
+ say "Generating #{name} index"
+
+ Gem.time "Generated #{name} index" do
+ open(file, 'wb') do |io|
+ specs = index.map do |*spec|
+ # We have to splat here because latest_specs is an array,
+ # while the others are hashes. See the TODO in source_index.rb
+ spec = spec.flatten.last
+ platform = spec.original_platform
+
+ # win32-api-1.0.4-x86-mswin32-60
+ unless String === platform then
+ alert_warning "Skipping invalid platform in gem: #{spec.full_name}"
+ next
+ end
+
+ platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
+ [spec.name, spec.version, platform]
+ end
+
+ specs = compact_specs(specs)
+ Marshal.dump(specs, io)
+ end
+ end
+ end
+
+ ##
+ # Builds indicies for RubyGems 1.2 and newer. Handles full, latest, prerelease
+
+ def build_modern_indicies(index)
+ build_modern_index(index.sort, @specs_index, 'specs')
+ build_modern_index(index.latest_specs.sort,
+ @latest_specs_index,
+ 'latest specs')
+ build_modern_index(index.prerelease_specs.sort,
+ @prerelease_specs_index,
+ 'prerelease specs')
+
+ @files += [@specs_index,
+ "#{@specs_index}.gz",
+ @latest_specs_index,
+ "#{@latest_specs_index}.gz",
+ @prerelease_specs_index,
+ "#{@prerelease_specs_index}.gz"]
+ end
- compress quick_index, 'rz'
- paranoid quick_index, 'rz'
+ ##
+ # Builds an RSS feed for past two days gem releases according to the gem's
+ # date.
- compress latest_index, 'rz'
- paranoid latest_index, 'rz'
+ def build_rss(index)
+ if @rss_host.nil? or @rss_gems_host.nil? then
+ if Gem.configuration.really_verbose then
+ alert_warning "no --rss-host or --rss-gems-host, RSS generation disabled"
+ end
+ return
+ end
- compress @marshal_index, 'Z'
- paranoid @marshal_index, 'Z'
+ require 'cgi'
+ require 'rubygems/text'
+
+ extend Gem::Text
+
+ Gem.time 'Generated rss' do
+ open @rss_index, 'wb' do |io|
+ rss_host = CGI.escapeHTML @rss_host
+ rss_title = CGI.escapeHTML(@rss_title || 'gems')
+
+ io.puts <<-HEADER
+<?xml version="1.0"?>
+<rss version="2.0">
+ <channel>
+ <title>#{rss_title}</title>
+ <link>http://#{rss_host}</link>
+ <description>Recently released gems from http://#{rss_host}</description>
+ <generator>RubyGems v#{Gem::RubyGemsVersion}</generator>
+ <docs>http://cyber.law.harvard.edu/rss/rss.html</docs>
+ HEADER
+
+ today = Gem::Specification::TODAY
+ yesterday = today - 86400
+
+ index = index.select do |_, spec|
+ spec_date = spec.date
+
+ case spec_date
+ when Date
+ Time.parse(spec_date.to_s) >= yesterday
+ when Time
+ spec_date >= yesterday
+ end
+ end
+
+ index = index.select do |_, spec|
+ spec_date = spec.date
- compress @master_index, 'Z'
- paranoid @master_index, 'Z'
+ case spec_date
+ when Date
+ Time.parse(spec_date.to_s) <= today
+ when Time
+ spec_date <= today
+ end
+ end
- gzip @specs_index
- gzip @latest_specs_index
+ index.sort_by { |_, spec| [-spec.date.to_i, spec] }.each do |_, spec|
+ gem_path = CGI.escapeHTML "http://#{@rss_gems_host}/gems/#{spec.full_name}.gem"
+ size = File.stat(spec.loaded_from).size rescue next
+
+ description = spec.description || spec.summary || ''
+ authors = Array spec.authors
+ emails = Array spec.email
+ authors = emails.zip(authors).map do |email, author|
+ email += " (#{author})" if author and not author.empty?
+ end.join ', '
+
+ description = description.split(/\n\n+/).map do |chunk|
+ format_text chunk, 78
+ end
+
+ description = description.join "\n\n"
+
+ item = ''
+
+ item << <<-ITEM
+ <item>
+ <title>#{CGI.escapeHTML spec.full_name}</title>
+ <description>
+&lt;pre&gt;#{CGI.escapeHTML description.chomp}&lt;/pre&gt;
+ </description>
+ <author>#{CGI.escapeHTML authors}</author>
+ <guid>#{CGI.escapeHTML spec.full_name}</guid>
+ <enclosure url=\"#{gem_path}\"
+ length=\"#{size}\" type=\"application/octet-stream\" />
+ <pubDate>#{spec.date.rfc2822}</pubDate>
+ ITEM
+
+ item << <<-ITEM if spec.homepage
+ <link>#{CGI.escapeHTML spec.homepage}</link>
+ ITEM
+
+ item << <<-ITEM
+ </item>
+ ITEM
+
+ io.puts item
+ end
+
+ io.puts <<-FOOTER
+ </channel>
+</rss>
+ FOOTER
+ end
+ end
+
+ @files << @rss_index
end
##
# Collect specifications from .gem files from the gem directory.
- def collect_specs
+ def collect_specs(gems = gem_file_list)
index = Gem::SourceIndex.new
- progress = ui.progress_reporter gem_file_list.size,
- "Loading #{gem_file_list.size} gems from #{@dest_directory}",
+ progress = ui.progress_reporter gems.size,
+ "Loading #{gems.size} gems from #{@dest_directory}",
"Loaded all gems"
- gem_file_list.each do |gemfile|
- if File.size(gemfile.to_s) == 0 then
- alert_warning "Skipping zero-length gem: #{gemfile}"
- next
- end
-
- begin
- spec = Gem::Format.from_file_by_path(gemfile).spec
-
- unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
- alert_warning "Skipping misnamed gem: #{gemfile} => #{spec.full_name} (#{spec.original_name})"
+ Gem.time 'loaded' do
+ gems.each do |gemfile|
+ if File.size(gemfile.to_s) == 0 then
+ alert_warning "Skipping zero-length gem: #{gemfile}"
next
end
- abbreviate spec
- sanitize spec
+ begin
+ spec = Gem::Format.from_file_by_path(gemfile).spec
+ spec.loaded_from = gemfile
+
+ unless gemfile =~ /\/#{Regexp.escape spec.original_name}.*\.gem\z/i then
+ expected_name = spec.full_name
+ expected_name << " (#{spec.original_name})" if
+ spec.original_name != spec.full_name
+ alert_warning "Skipping misnamed gem: #{gemfile} should be named #{expected_name}"
+ next
+ end
- index.gems[spec.original_name] = spec
+ abbreviate spec
+ sanitize spec
- progress.updated spec.original_name
+ index.add_spec spec, spec.original_name
- rescue SignalException => e
- alert_error "Received signal, exiting"
- raise
- rescue Exception => e
- alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
+ progress.updated spec.original_name
+
+ rescue SignalException => e
+ alert_error "Received signal, exiting"
+ raise
+ rescue Exception => e
+ alert_error "Unable to process #{gemfile}\n#{e.message} (#{e.class})\n\t#{e.backtrace.join "\n\t"}"
+ end
end
- end
- progress.done
+ progress.done
+ end
index
end
##
+ # Compresses indicies on disk
+ #--
+ # All future files should be compressed using gzip, not deflate
+
+ def compress_indicies
+ say "Compressing indicies"
+
+ Gem.time 'Compressed indicies' do
+ if @build_legacy then
+ compress @quick_index, 'rz'
+ paranoid @quick_index, 'rz'
+
+ compress @latest_index, 'rz'
+ paranoid @latest_index, 'rz'
+
+ compress @marshal_index, 'Z'
+ paranoid @marshal_index, 'Z'
+
+ compress @master_index, 'Z'
+ paranoid @master_index, 'Z'
+ end
+
+ if @build_modern then
+ gzip @specs_index
+ gzip @latest_specs_index
+ gzip @prerelease_specs_index
+ end
+ end
+ end
+
+ ##
# Compacts Marshal output for the specs index data source by using identical
# objects as much as possible.
@@ -282,7 +522,7 @@ class Gem::Indexer
end
##
- # Builds and installs indexicies.
+ # Builds and installs indicies.
def generate_index
make_temp_directories
@@ -311,7 +551,27 @@ class Gem::Indexer
say "Moving index into production dir #{@dest_directory}" if verbose
- @files.each do |file|
+ files = @files.dup
+ files.delete @quick_marshal_dir if files.include? @quick_dir
+
+ if files.include? @quick_marshal_dir and
+ not files.include? @quick_dir then
+ files.delete @quick_marshal_dir
+ quick_marshal_dir = @quick_marshal_dir.sub @directory, ''
+
+ dst_name = File.join @dest_directory, quick_marshal_dir
+
+ FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose
+ FileUtils.rm_rf dst_name, :verbose => verbose
+ FileUtils.mv @quick_marshal_dir, dst_name, :verbose => verbose,
+ :force => true
+ end
+
+ files = files.map do |path|
+ path.sub @directory, ''
+ end
+
+ files.each do |file|
src_name = File.join @directory, file
dst_name = File.join @dest_directory, file
@@ -366,5 +626,87 @@ class Gem::Indexer
string ? string.to_s.to_xs : string
end
+ ##
+ # Perform an in-place update of the repository from newly added gems. Only
+ # works for modern indicies, and sets #build_legacy to false when run.
+
+ def update_index
+ @build_legacy = false
+
+ make_temp_directories
+
+ specs_mtime = File.stat(@dest_specs_index).mtime
+ newest_mtime = Time.at 0
+
+ updated_gems = gem_file_list.select do |gem|
+ gem_mtime = File.stat(gem).mtime
+ newest_mtime = gem_mtime if gem_mtime > newest_mtime
+ gem_mtime >= specs_mtime
+ end
+
+ if updated_gems.empty? then
+ say 'No new gems'
+ terminate_interaction 0
+ end
+
+ index = collect_specs updated_gems
+
+ files = build_marshal_gemspecs index
+
+ Gem.time 'Updated indexes' do
+ update_specs_index index, @dest_specs_index, @specs_index
+ update_specs_index index, @dest_latest_specs_index, @latest_specs_index
+ update_specs_index(index.prerelease_gems, @dest_prerelease_specs_index,
+ @prerelease_specs_index)
+ end
+
+ compress_indicies
+
+ verbose = Gem.configuration.really_verbose
+
+ say "Updating production dir #{@dest_directory}" if verbose
+
+ files << @specs_index
+ files << "#{@specs_index}.gz"
+ files << @latest_specs_index
+ files << "#{@latest_specs_index}.gz"
+ files << @prerelease_specs_index
+ files << "#{@prerelease_specs_index}.gz"
+
+ files = files.map do |path|
+ path.sub @directory, ''
+ end
+
+ files.each do |file|
+ src_name = File.join @directory, file
+ dst_name = File.join @dest_directory, File.dirname(file)
+
+ FileUtils.mv src_name, dst_name, :verbose => verbose,
+ :force => true
+
+ File.utime newest_mtime, newest_mtime, dst_name
+ end
+ end
+
+ ##
+ # Combines specs in +index+ and +source+ then writes out a new copy to
+ # +dest+. For a latest index, does not ensure the new file is minimal.
+
+ def update_specs_index(index, source, dest)
+ specs_index = Marshal.load Gem.read_binary(source)
+
+ index.each do |_, spec|
+ platform = spec.original_platform
+ platform = Gem::Platform::RUBY if platform.nil? or platform.empty?
+ specs_index << [spec.name, spec.version, platform]
+ end
+
+ specs_index = compact_specs specs_index.uniq.sort
+
+ open dest, 'wb' do |io|
+ Marshal.dump specs_index, io
+ end
+ end
+
end
generated by cgit v1.2.3 (git 2.39.1) at 2024-05-25 07:36:39 +0000