diff options
author | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-09-08 08:45:41 +0000 |
---|---|---|
committer | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-09-08 08:45:41 +0000 |
commit | 8598f8c2dc78c6d1ae87cb6ae19c34ba2cb29241 (patch) | |
tree | 0bbd28f684e745cb212761b7c74fe08668f85cc8 /lib/bundler/fetcher | |
parent | f2e04b77aa8a363d7e36ce5a9cdb60714a537a3c (diff) | |
download | ruby-8598f8c2dc78c6d1ae87cb6ae19c34ba2cb29241.tar.gz |
Merge bundler to standard libraries.
rubygems 2.7.x depends bundler-1.15.x. This is preparation for
rubygems and bundler migration.
* lib/bundler.rb, lib/bundler/*: files of bundler-1.15.4
* spec/bundler/*: rspec examples of bundler-1.15.4. I applied patches.
* https://github.com/bundler/bundler/pull/6007
* Exclude not working examples on ruby repository.
* Fake ruby interpriter instead of installed ruby.
* Makefile.in: Added test task named `test-bundler`. This task is only
working macOS/linux yet. I'm going to support Windows environment later.
* tool/sync_default_gems.rb: Added sync task for bundler.
[Feature #12733][ruby-core:77172]
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59779 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/bundler/fetcher')
-rw-r--r-- | lib/bundler/fetcher/base.rb | 51 | ||||
-rw-r--r-- | lib/bundler/fetcher/compact_index.rb | 135 | ||||
-rw-r--r-- | lib/bundler/fetcher/dependency.rb | 81 | ||||
-rw-r--r-- | lib/bundler/fetcher/downloader.rb | 78 | ||||
-rw-r--r-- | lib/bundler/fetcher/index.rb | 51 |
5 files changed, 396 insertions, 0 deletions
diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb new file mode 100644 index 0000000000..271729a534 --- /dev/null +++ b/lib/bundler/fetcher/base.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true +module Bundler + class Fetcher + class Base + attr_reader :downloader + attr_reader :display_uri + attr_reader :remote + + def initialize(downloader, remote, display_uri) + raise "Abstract class" if self.class == Base + @downloader = downloader + @remote = remote + @display_uri = display_uri + end + + def remote_uri + @remote.uri + end + + def fetch_uri + @fetch_uri ||= begin + if remote_uri.host == "rubygems.org" + uri = remote_uri.dup + uri.host = "index.rubygems.org" + uri + else + remote_uri + end + end + end + + def available? + true + end + + def api_fetcher? + false + end + + private + + def log_specs(debug_msg) + if Bundler.ui.debug? + Bundler.ui.debug debug_msg + else + Bundler.ui.info ".", false + end + end + end + end +end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb new file mode 100644 index 0000000000..97de88101b --- /dev/null +++ b/lib/bundler/fetcher/compact_index.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true +require "bundler/fetcher/base" +require "bundler/worker" + +module Bundler + autoload :CompactIndexClient, "bundler/compact_index_client" + + class Fetcher + class CompactIndex < Base + def self.compact_index_request(method_name) + method = instance_method(method_name) + undef_method(method_name) + define_method(method_name) do |*args, &blk| + begin + method.bind(self).call(*args, &blk) + rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e + raise HTTPError, e.message + rescue AuthenticationRequiredError + # Fail since we got a 401 from the server. + raise + rescue HTTPError => e + Bundler.ui.trace(e) + nil + end + end + end + + def specs(gem_names) + specs_for_names(gem_names) + end + compact_index_request :specs + + def specs_for_names(gem_names) + gem_info = [] + complete_gems = [] + remaining_gems = gem_names.dup + + until remaining_gems.empty? + log_specs "Looking up gems #{remaining_gems.inspect}" + + deps = compact_index_client.dependencies(remaining_gems) + next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq + deps.each {|dep| gem_info << dep } + complete_gems.concat(deps.map(&:first)).uniq! + remaining_gems = next_gems - complete_gems + end + @bundle_worker.stop if @bundle_worker + @bundle_worker = nil # reset it. Not sure if necessary + + gem_info + end + + def fetch_spec(spec) + spec -= [nil, "ruby", ""] + contents = compact_index_client.spec(*spec) + return nil if contents.nil? + contents.unshift(spec.first) + contents[3].map! {|d| Gem::Dependency.new(*d) } + EndpointSpecification.new(*contents) + end + compact_index_request :fetch_spec + + def available? + return nil unless md5_available? + user_home = Bundler.user_home + return nil unless user_home.directory? && user_home.writable? + # Read info file checksums out of /versions, so we can know if gems are up to date + fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums! + rescue CompactIndexClient::Updater::MisMatchedChecksumError => e + Bundler.ui.debug(e.message) + nil + end + compact_index_request :available? + + def api_fetcher? + true + end + + private + + def compact_index_client + @compact_index_client ||= begin + SharedHelpers.filesystem_access(cache_path) do + CompactIndexClient.new(cache_path, client_fetcher) + end.tap do |client| + client.in_parallel = lambda do |inputs, &blk| + func = lambda {|object, _index| blk.call(object) } + worker = bundle_worker(func) + inputs.each {|input| worker.enq(input) } + inputs.map { worker.deq } + end + end + end + end + + def bundle_worker(func = nil) + @bundle_worker ||= begin + worker_name = "Compact Index (#{display_uri.host})" + Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func) + end + @bundle_worker.tap do |worker| + worker.instance_variable_set(:@func, func) if func + end + end + + def cache_path + Bundler.user_cache.join("compact_index", remote.cache_slug) + end + + def client_fetcher + ClientFetcher.new(self, Bundler.ui) + end + + ClientFetcher = Struct.new(:fetcher, :ui) do + def call(path, headers) + fetcher.downloader.fetch(fetcher.fetch_uri + path, headers) + rescue NetworkDownError => e + raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"] + ui.warn "Using the cached data for the new index because of a network error: #{e}" + Net::HTTPNotModified.new(nil, nil, nil) + end + end + + def md5_available? + require "openssl" + OpenSSL::Digest::MD5.digest("") + true + rescue LoadError + true + rescue OpenSSL::Digest::DigestError + false + end + end + end +end diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb new file mode 100644 index 0000000000..445b0f2332 --- /dev/null +++ b/lib/bundler/fetcher/dependency.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true +require "bundler/fetcher/base" +require "cgi" + +module Bundler + class Fetcher + class Dependency < Base + def available? + fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri) + rescue NetworkDownError => e + raise HTTPError, e.message + rescue AuthenticationRequiredError + # Fail since we got a 401 from the server. + raise + rescue HTTPError + false + end + + def api_fetcher? + true + end + + def specs(gem_names, full_dependency_list = [], last_spec_list = []) + query_list = gem_names.uniq - full_dependency_list + + log_specs "Query List: #{query_list.inspect}" + + return last_spec_list if query_list.empty? + + spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do + dependency_specs(query_list) + end + + returned_gems = spec_list.map(&:first).uniq + specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list) + rescue MarshalError + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + Bundler.ui.debug "could not fetch from the dependency API, trying the full index" + nil + rescue HTTPError, GemspecError + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + Bundler.ui.debug "could not fetch from the dependency API\nit's suggested to retry using the full index via `bundle install --full-index`" + nil + end + + def dependency_specs(gem_names) + Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}" + + gem_list = unmarshalled_dep_gems(gem_names) + get_formatted_specs_and_deps(gem_list) + end + + def unmarshalled_dep_gems(gem_names) + gem_list = [] + gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names| + marshalled_deps = downloader.fetch(dependency_api_uri(names)).body + gem_list.concat(Bundler.load_marshal(marshalled_deps)) + end + gem_list + end + + def get_formatted_specs_and_deps(gem_list) + deps_list = [] + spec_list = [] + + gem_list.each do |s| + deps_list.concat(s[:dependencies].map(&:first)) + deps = s[:dependencies].map {|n, d| [n, d.split(", ")] } + spec_list.push([s[:name], s[:number], s[:platform], deps]) + end + [spec_list, deps_list] + end + + def dependency_api_uri(gem_names = []) + uri = fetch_uri + "api/v1/dependencies" + uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any? + uri + end + end + end +end diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb new file mode 100644 index 0000000000..453e4645eb --- /dev/null +++ b/lib/bundler/fetcher/downloader.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true +module Bundler + class Fetcher + class Downloader + attr_reader :connection + attr_reader :redirect_limit + + def initialize(connection, redirect_limit) + @connection = connection + @redirect_limit = redirect_limit + end + + def fetch(uri, options = {}, counter = 0) + raise HTTPError, "Too many redirects" if counter >= redirect_limit + + response = request(uri, options) + Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}") + + case response + when Net::HTTPSuccess, Net::HTTPNotModified + response + when Net::HTTPRedirection + new_uri = URI.parse(response["location"]) + if new_uri.host == uri.host + new_uri.user = uri.user + new_uri.password = uri.password + end + fetch(new_uri, options, counter + 1) + when Net::HTTPRequestEntityTooLarge + raise FallbackError, response.body + when Net::HTTPUnauthorized + raise AuthenticationRequiredError, uri.host + when Net::HTTPNotFound + raise FallbackError, "Net::HTTPNotFound" + else + raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}" + end + end + + def request(uri, options) + validate_uri_scheme!(uri) + + Bundler.ui.debug "HTTP GET #{uri}" + req = Net::HTTP::Get.new uri.request_uri, options + if uri.user + user = CGI.unescape(uri.user) + password = uri.password ? CGI.unescape(uri.password) : nil + req.basic_auth(user, password) + end + connection.request(uri, req) + rescue NoMethodError => e + raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet } + raise LoadError.new("cannot load such file -- openssl") + rescue OpenSSL::SSL::SSLError + raise CertificateFailureError.new(uri) + rescue *HTTP_ERRORS => e + Bundler.ui.trace e + case e.message + when /host down:/, /getaddrinfo: nodename nor servname provided/ + raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \ + "connection and try again." + else + raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \ + " (#{e})" + end + end + + private + + def validate_uri_scheme!(uri) + return if uri.scheme =~ /\Ahttps?\z/ + raise InvalidOption, + "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \ + "Did you mean `http` or `https`?" + end + end + end +end diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb new file mode 100644 index 0000000000..d8e212989e --- /dev/null +++ b/lib/bundler/fetcher/index.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true +require "bundler/fetcher/base" +require "rubygems/remote_fetcher" + +module Bundler + class Fetcher + class Index < Base + def specs(_gem_names) + Bundler.rubygems.fetch_all_remote_specs(remote) + rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e + case e.message + when /certificate verify failed/ + raise CertificateFailureError.new(display_uri) + when /401/ + raise AuthenticationRequiredError, remote_uri + when /403/ + raise BadAuthenticationError, remote_uri if remote_uri.userinfo + raise AuthenticationRequiredError, remote_uri + else + Bundler.ui.trace e + raise HTTPError, "Could not fetch specs from #{display_uri}" + end + end + + def fetch_spec(spec) + spec -= [nil, "ruby", ""] + spec_file_name = "#{spec.join "-"}.gemspec" + + uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") + if uri.scheme == "file" + Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path)) + elsif cached_spec_path = gemspec_cached_path(spec_file_name) + Bundler.load_gemspec(cached_spec_path) + else + Bundler.load_marshal Gem.inflate(downloader.fetch(uri).body) + end + rescue MarshalError + raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \ + "Your network or your gem server is probably having issues right now." + end + + private + + # cached gem specification path, if one exists + def gemspec_cached_path(spec_file_name) + paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) } + paths.find {|path| File.file? path } + end + end + end +end |