diff options
author | Samuel Giddins <segiddins@segiddins.me> | 2023-09-18 13:56:20 -0700 |
---|---|---|
committer | git <svn-admin@ruby-lang.org> | 2023-10-08 04:17:15 +0000 |
commit | 2b6228be48dc656c3e7a95919622e4b8561324cf (patch) | |
tree | d964da45ed6fa3930c234c872eb52de0a5b26858 | |
parent | bf71b0eda54b551db023cda9051b9be218c0f75d (diff) | |
download | ruby-2b6228be48dc656c3e7a95919622e4b8561324cf.tar.gz |
[rubygems/rubygems] Reduce allocations when parsing compact index
This still allocates a ton (a string for each line, plus a bunch of
splits into arrays), but it helps a bit when Bundler has to go through
dependency resolution.
```
==> memprof.after.txt <==
Total allocated: 194.14 MB (2317172 objects)
Total retained: 60.81 MB (593164 objects)
==> memprof.before.txt <==
Total allocated: 211.97 MB (2404890 objects)
Total retained: 62.85 MB (640342 objects)
```
https://github.com/rubygems/rubygems/commit/c68b41b0e5
-rw-r--r-- | lib/bundler/compact_index_client/cache.rb | 6 | ||||
-rw-r--r-- | lib/bundler/compact_index_client/gem_parser.rb | 10 | ||||
-rw-r--r-- | lib/bundler/compact_index_client/updater.rb | 7 | ||||
-rw-r--r-- | lib/rubygems/resolver/api_set/gem_parser.rb | 10 |
4 files changed, 23 insertions, 10 deletions
diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index 0b43581c11..b5607c8751 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -32,11 +32,11 @@ module Bundler name, versions_string, info_checksum = line.split(" ", 3) info_checksums_by_name[name] = info_checksum || "" versions_string.split(",").each do |version| - if version.start_with?("-") - version = version[1..-1].split("-", 2).unshift(name) + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete versions_by_name[name].delete(version) else - version = version.split("-", 2).unshift(name) versions_by_name[name] << version end end diff --git a/lib/bundler/compact_index_client/gem_parser.rb b/lib/bundler/compact_index_client/gem_parser.rb index e7bf4c6001..60a1817607 100644 --- a/lib/bundler/compact_index_client/gem_parser.rb +++ b/lib/bundler/compact_index_client/gem_parser.rb @@ -6,12 +6,15 @@ module Bundler GemParser = Gem::Resolver::APISet::GemParser else class GemParser + EMPTY_ARRAY = [].freeze + private_constant :EMPTY_ARRAY + def parse(line) version_and_platform, rest = line.split(" ", 2) version, platform = version_and_platform.split("-", 2) - dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest - dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] - requirements = requirements ? requirements.map {|d| parse_dependency(d) } : [] + dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest + dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY + requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY [version, platform, dependencies, requirements] end @@ -20,6 +23,7 @@ module Bundler def parse_dependency(string) dependency = string.split(":") dependency[-1] = dependency[-1].split("&") if dependency.size > 1 + dependency[0] = -dependency[0] dependency end end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 0f7bf9bb50..3b75d5c129 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -95,7 +95,12 @@ module Bundler # because we need to preserve \n line endings on windows when calculating # the checksum SharedHelpers.filesystem_access(path, :read) do - SharedHelpers.digest(:MD5).hexdigest(File.read(path)) + File.open(path, "rb") do |f| + digest = SharedHelpers.digest(:MD5).new + buf = String.new(:capacity => 16_384, :encoding => Encoding::BINARY) + digest << buf while f.read(16_384, buf) + digest.hexdigest + end end end diff --git a/lib/rubygems/resolver/api_set/gem_parser.rb b/lib/rubygems/resolver/api_set/gem_parser.rb index 685c39558d..643b857107 100644 --- a/lib/rubygems/resolver/api_set/gem_parser.rb +++ b/lib/rubygems/resolver/api_set/gem_parser.rb @@ -1,12 +1,15 @@ # frozen_string_literal: true class Gem::Resolver::APISet::GemParser + EMPTY_ARRAY = [].freeze + private_constant :EMPTY_ARRAY + def parse(line) version_and_platform, rest = line.split(" ", 2) version, platform = version_and_platform.split("-", 2) - dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest - dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] - requirements = requirements ? requirements.map {|d| parse_dependency(d) } : [] + dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest + dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY + requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY [version, platform, dependencies, requirements] end @@ -15,6 +18,7 @@ class Gem::Resolver::APISet::GemParser def parse_dependency(string) dependency = string.split(":") dependency[-1] = dependency[-1].split("&") if dependency.size > 1 + dependency[0] = -dependency[0] dependency end end |