1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
|
#--
# Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others.
# All rights reserved.
# See LICENSE.txt for permissions.
#++
require 'rubygems/user_interaction'
require 'rubygems/specification'
# :stopdoc:
module Gem
autoload :SpecFetcher, 'rubygems/spec_fetcher'
end
# :startdoc:
##
# The SourceIndex object indexes all the gems available from a
# particular source (e.g. a list of gem directories, or a remote
# source). A SourceIndex maps a gem full name to a gem
# specification.
#
# NOTE:: The class used to be named Cache, but that became
# confusing when cached source fetchers where introduced. The
# constant Gem::Cache is an alias for this class to allow old
# YAMLized source index objects to load properly.
class Gem::SourceIndex
include Enumerable
include Gem::UserInteraction
attr_reader :gems, :prerelease_gems # :nodoc:
##
# Directories to use to refresh this SourceIndex when calling refresh!
attr_accessor :spec_dirs
class << self
include Gem::UserInteraction
##
# Factory method to construct a source index instance for a given
# path.
#
# deprecated::
# If supplied, from_installed_gems will act just like
# +from_gems_in+. This argument is deprecated and is provided
# just for backwards compatibility, and should not generally
# be used.
#
# return::
# SourceIndex instance
def from_installed_gems(*deprecated)
if deprecated.empty?
from_gems_in(*installed_spec_directories)
else
from_gems_in(*deprecated) # HACK warn
end
end
##
# Returns a list of directories from Gem.path that contain specifications.
def installed_spec_directories
Gem.path.collect { |dir| File.join(dir, "specifications") }
end
##
# Creates a new SourceIndex from the ruby format gem specifications in
# +spec_dirs+.
def from_gems_in(*spec_dirs)
source_index = new
source_index.spec_dirs = spec_dirs
source_index.refresh!
end
##
# Loads a ruby-format specification from +file_name+ and returns the
# loaded spec.
def load_specification(file_name)
return nil unless file_name and File.exist? file_name
spec_code = if RUBY_VERSION < '1.9' then
File.read file_name
else
File.read file_name, :encoding => 'UTF-8'
end.untaint
begin
gemspec = eval spec_code, binding, file_name
if gemspec.is_a?(Gem::Specification)
gemspec.loaded_from = file_name
return gemspec
end
alert_warning "File '#{file_name}' does not evaluate to a gem specification"
rescue SignalException, SystemExit
raise
rescue SyntaxError => e
alert_warning e
alert_warning spec_code
rescue Exception => e
alert_warning "#{e.inspect}\n#{spec_code}"
alert_warning "Invalid .gemspec format in '#{file_name}'"
end
return nil
end
end
##
# Constructs a source index instance from the provided specifications, which
# is a Hash of gem full names and Gem::Specifications.
#--
# TODO merge @gems and @prerelease_gems and provide a separate method
# #prerelease_gems
def initialize(specifications={})
@gems, @prerelease_gems = [{}, {}]
specifications.each{ |full_name, spec| add_spec spec }
@spec_dirs = nil
end
##
# Both regular and prerelease gems
def all_gems
@gems.merge @prerelease_gems
end
##
# Reconstruct the source index from the specifications in +spec_dirs+.
def load_gems_in(*spec_dirs)
@gems.clear
spec_dirs.reverse_each do |spec_dir|
spec_files = Dir.glob File.join(spec_dir, '*.gemspec')
spec_files.each do |spec_file|
gemspec = self.class.load_specification spec_file.untaint
add_spec gemspec if gemspec
end
end
self
end
##
# Returns an Array specifications for the latest versions of each gem in
# this index.
def latest_specs
result = Hash.new { |h,k| h[k] = [] }
latest = {}
sort.each do |_, spec|
name = spec.name
curr_ver = spec.version
prev_ver = latest.key?(name) ? latest[name].version : nil
next unless prev_ver.nil? or curr_ver >= prev_ver or
latest[name].platform != Gem::Platform::RUBY
if prev_ver.nil? or
(curr_ver > prev_ver and spec.platform == Gem::Platform::RUBY) then
result[name].clear
latest[name] = spec
end
if spec.platform != Gem::Platform::RUBY then
result[name].delete_if do |result_spec|
result_spec.platform == spec.platform
end
end
result[name] << spec
end
# TODO: why is this a hash while @gems is an array? Seems like
# structural similarity would be good.
result.values.flatten
end
##
# An array including only the prerelease gemspecs
def prerelease_specs
@prerelease_gems.values
end
##
# Add a gem specification to the source index.
def add_spec(gem_spec, name = gem_spec.full_name)
# No idea why, but the Indexer wants to insert them using original_name
# instead of full_name. So we make it an optional arg.
if gem_spec.version.prerelease?
@prerelease_gems[name] = gem_spec
else
@gems[name] = gem_spec
end
end
##
# Add gem specifications to the source index.
def add_specs(*gem_specs)
gem_specs.each do |spec|
add_spec spec
end
end
##
# Remove a gem specification named +full_name+.
def remove_spec(full_name)
if @gems.key? full_name then
@gems.delete full_name
else
@prerelease_gems.delete full_name
end
end
##
# Iterate over the specifications in the source index.
def each(&block) # :yields: gem.full_name, gem
@gems.each(&block)
end
##
# The gem specification given a full gem spec name.
def specification(full_name)
@gems[full_name]
end
##
# The signature for the source index. Changes in the signature indicate a
# change in the index.
def index_signature
require 'digest'
Digest::SHA256.new.hexdigest(@gems.keys.sort.join(',')).to_s
end
##
# The signature for the given gem specification.
def gem_signature(gem_full_name)
require 'digest'
Digest::SHA256.new.hexdigest(@gems[gem_full_name].to_yaml).to_s
end
def size
@gems.size
end
alias length size
##
# Find a gem by an exact match on the short name.
def find_name(gem_name, version_requirement = Gem::Requirement.default)
dep = Gem::Dependency.new gem_name, version_requirement
search dep
end
##
# Search for a gem by Gem::Dependency +gem_pattern+. If +only_platform+
# is true, only gems matching Gem::Platform.local will be returned. An
# Array of matching Gem::Specification objects is returned.
#
# For backwards compatibility, a String or Regexp pattern may be passed as
# +gem_pattern+, and a Gem::Requirement for +platform_only+. This
# behavior is deprecated and will be removed.
def search(gem_pattern, platform_only = false)
version_requirement = nil
only_platform = false
# TODO - Remove support and warning for legacy arguments after 2008/11
unless Gem::Dependency === gem_pattern
warn "#{Gem.location_of_caller.join ':'}:Warning: Gem::SourceIndex#search support for #{gem_pattern.class} patterns is deprecated, use #find_name"
end
case gem_pattern
when Regexp then
version_requirement = platform_only || Gem::Requirement.default
when Gem::Dependency then
only_platform = platform_only
version_requirement = gem_pattern.version_requirements
gem_pattern = if Regexp === gem_pattern.name then
gem_pattern.name
elsif gem_pattern.name.empty? then
//
else
/^#{Regexp.escape gem_pattern.name}$/
end
else
version_requirement = platform_only || Gem::Requirement.default
gem_pattern = /#{gem_pattern}/i
end
unless Gem::Requirement === version_requirement then
version_requirement = Gem::Requirement.create version_requirement
end
specs = all_gems.values.select do |spec|
spec.name =~ gem_pattern and
version_requirement.satisfied_by? spec.version
end
if only_platform then
specs = specs.select do |spec|
Gem::Platform.match spec.platform
end
end
specs.sort_by { |s| s.sort_obj }
end
##
# Replaces the gems in the source index from specifications in the
# directories this source index was created from. Raises an exception if
# this source index wasn't created from a directory (via from_gems_in or
# from_installed_gems, or having spec_dirs set).
def refresh!
raise 'source index not created from disk' if @spec_dirs.nil?
load_gems_in(*@spec_dirs)
end
##
# Returns an Array of Gem::Specifications that are not up to date.
def outdated
outdateds = []
latest_specs.each do |local|
dependency = Gem::Dependency.new local.name, ">= #{local.version}"
begin
fetcher = Gem::SpecFetcher.fetcher
remotes = fetcher.find_matching dependency
remotes = remotes.map { |(name, version,_),_| version }
rescue Gem::RemoteFetcher::FetchError => e
raise unless fetcher.warn_legacy e do
require 'rubygems/source_info_cache'
specs = Gem::SourceInfoCache.search_with_source dependency, true
remotes = specs.map { |spec,| spec.version }
end
end
latest = remotes.sort.last
outdateds << local.name if latest and local.version < latest
end
outdateds
end
##
# Updates this SourceIndex from +source_uri+. If +all+ is false, only the
# latest gems are fetched.
def update(source_uri, all)
source_uri = URI.parse source_uri unless URI::Generic === source_uri
source_uri.path += '/' unless source_uri.path =~ /\/$/
use_incremental = false
begin
gem_names = fetch_quick_index source_uri, all
remove_extra gem_names
missing_gems = find_missing gem_names
return false if missing_gems.size.zero?
say "Missing metadata for #{missing_gems.size} gems" if
missing_gems.size > 0 and Gem.configuration.really_verbose
use_incremental = missing_gems.size <= Gem.configuration.bulk_threshold
rescue Gem::OperationNotSupportedError => ex
alert_error "Falling back to bulk fetch: #{ex.message}" if
Gem.configuration.really_verbose
use_incremental = false
end
if use_incremental then
update_with_missing(source_uri, missing_gems)
else
new_index = fetch_bulk_index(source_uri)
@gems.replace(new_index.gems)
end
true
end
def ==(other) # :nodoc:
self.class === other and @gems == other.gems
end
def dump
Marshal.dump(self)
end
private
def fetcher
require 'rubygems/remote_fetcher'
Gem::RemoteFetcher.fetcher
end
def fetch_index_from(source_uri)
@fetch_error = nil
indexes = %W[
Marshal.#{Gem.marshal_version}.Z
Marshal.#{Gem.marshal_version}
yaml.Z
yaml
]
indexes.each do |name|
spec_data = nil
index = source_uri + name
begin
spec_data = fetcher.fetch_path index
spec_data = unzip(spec_data) if name =~ /\.Z$/
if name =~ /Marshal/ then
return Marshal.load(spec_data)
else
return YAML.load(spec_data)
end
rescue => e
if Gem.configuration.really_verbose then
alert_error "Unable to fetch #{name}: #{e.message}"
end
@fetch_error = e
end
end
nil
end
def fetch_bulk_index(source_uri)
say "Bulk updating Gem source index for: #{source_uri}" if
Gem.configuration.verbose
index = fetch_index_from(source_uri)
if index.nil? then
raise Gem::RemoteSourceException,
"Error fetching remote gem cache: #{@fetch_error}"
end
@fetch_error = nil
index
end
##
# Get the quick index needed for incremental updates.
def fetch_quick_index(source_uri, all)
index = all ? 'index' : 'latest_index'
zipped_index = fetcher.fetch_path source_uri + "quick/#{index}.rz"
unzip(zipped_index).split("\n")
rescue ::Exception => e
unless all then
say "Latest index not found, using quick index" if
Gem.configuration.really_verbose
fetch_quick_index source_uri, true
else
raise Gem::OperationNotSupportedError,
"No quick index found: #{e.message}"
end
end
##
# Make a list of full names for all the missing gemspecs.
def find_missing(spec_names)
unless defined? @originals then
@originals = {}
each do |full_name, spec|
@originals[spec.original_name] = spec
end
end
spec_names.find_all { |full_name|
@originals[full_name].nil?
}
end
def remove_extra(spec_names)
dictionary = spec_names.inject({}) { |h, k| h[k] = true; h }
each do |name, spec|
remove_spec name unless dictionary.include? spec.original_name
end
end
##
# Unzip the given string.
def unzip(string)
require 'zlib'
Gem.inflate string
end
##
# Tries to fetch Marshal representation first, then YAML
def fetch_single_spec(source_uri, spec_name)
@fetch_error = nil
begin
marshal_uri = source_uri + "quick/Marshal.#{Gem.marshal_version}/#{spec_name}.gemspec.rz"
zipped = fetcher.fetch_path marshal_uri
return Marshal.load(unzip(zipped))
rescue => ex
@fetch_error = ex
if Gem.configuration.really_verbose then
say "unable to fetch marshal gemspec #{marshal_uri}: #{ex.class} - #{ex}"
end
end
begin
yaml_uri = source_uri + "quick/#{spec_name}.gemspec.rz"
zipped = fetcher.fetch_path yaml_uri
return YAML.load(unzip(zipped))
rescue => ex
@fetch_error = ex
if Gem.configuration.really_verbose then
say "unable to fetch YAML gemspec #{yaml_uri}: #{ex.class} - #{ex}"
end
end
nil
end
##
# Update the cached source index with the missing names.
def update_with_missing(source_uri, missing_names)
progress = ui.progress_reporter(missing_names.size,
"Updating metadata for #{missing_names.size} gems from #{source_uri}")
missing_names.each do |spec_name|
gemspec = fetch_single_spec(source_uri, spec_name)
if gemspec.nil? then
ui.say "Failed to download spec #{spec_name} from #{source_uri}:\n" \
"\t#{@fetch_error.message}"
else
add_spec gemspec
progress.updated spec_name
end
@fetch_error = nil
end
progress.done
progress.count
end
end
# :stopdoc:
module Gem
##
# Cache is an alias for SourceIndex to allow older YAMLized source index
# objects to load properly.
Cache = SourceIndex
end
# :startdoc:
|