Opaque Etags for compact index requests

This changes the CompactIndexClient to store etags received from the
compact index in separate files rather than relying on the MD5 checksum
of the file as the etag.

Smoothes the upgrade from md5 etags to opaque by generating them when no
etag file exists. This should reduce the initial impact of changing the
caching behavior by reducing cache misses when the MD5 etag is the same.

Eventually, the MD5 behavior should be retired and the etag should be
considered completely opaque with no assumption that MD5 would match.

1 files changed, 105 insertions, 20 deletions

diff --git a/spec/bundler/install/gems/compact_index_spec.rb b/spec/bundler/install/gems/compact_index_spec.rb
index b2ed9565a4..b383614410 100644
--- a/spec/bundler/install/gems/compact_index_spec.rb
+++ b/spec/bundler/install/gems/compact_index_spec.rb
@@ -157,9 +157,8 @@ RSpec.describe "compact index api" do
 
     bundle :install, :verbose => true, :artifice => "compact_index_checksum_mismatch"
     expect(out).to include("Fetching gem metadata from #{source_uri}")
-    expect(out).to include <<-'WARN'
-The checksum of /versions does not match the checksum provided by the server! Something is wrong (local checksum is "\"d41d8cd98f00b204e9800998ecf8427e\"", was expecting "\"123\"").
-    WARN
+    expect(out).to include("The checksum of /versions does not match the checksum provided by the server!")
+    expect(out).to include('Calculated checksums {"sha-256"=>"8KfZiM/fszVkqhP/m5s9lvE6M9xKu4I1bU4Izddp5Ms="} did not match expected {"sha-256"=>"ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0="}')
     expect(the_bundle).to include_gems "rack 1.0.0"
   end
 
@@ -169,10 +168,12 @@ The checksum of /versions does not match the checksum provided by the server! So
       gem "rack"
     G
 
-    versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index",
-      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions")
-    FileUtils.mkdir_p(File.dirname(versions))
-    FileUtils.touch(versions)
+    versions = Pathname.new(Bundler.rubygems.user_home).join(
+      ".bundle", "cache", "compact_index",
+      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions"
+    )
+    versions.dirname.mkpath
+    versions.write("created_at")
     FileUtils.chmod("-r", versions)
 
     bundle :install, :artifice => "compact_index", :raise_on_error => false
@@ -772,23 +773,83 @@ The checksum of /versions does not match the checksum provided by the server! So
     end
   end
 
-  it "performs partial update with a non-empty range" do
+  it "performs update with etag not-modified" do
+    versions_etag = Pathname.new(Bundler.rubygems.user_home).join(
+      ".bundle", "cache", "compact_index",
+      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions.etag"
+    )
+    expect(versions_etag.file?).to eq(false)
+
     gemfile <<-G
       source "#{source_uri}"
       gem 'rack', '0.9.1'
     G
 
-    # Initial install creates the cached versions file
+    # Initial install creates the cached versions file and etag file
     bundle :install, :artifice => "compact_index"
 
+    expect(versions_etag.file?).to eq(true)
+    previous_content = versions_etag.binread
+
     # Update the Gemfile so we can check subsequent install was successful
     gemfile <<-G
       source "#{source_uri}"
       gem 'rack', '1.0.0'
     G
 
-    # Second install should make only a partial request to /versions
-    bundle :install, :artifice => "compact_index_partial_update"
+    # Second install should match etag
+    bundle :install, :artifice => "compact_index_etag_match"
+
+    expect(versions_etag.binread).to eq(previous_content)
+    expect(the_bundle).to include_gems "rack 1.0.0"
+  end
+
+  it "performs full update when range is ignored" do
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'rack', '0.9.1'
+    G
+
+    # Initial install creates the cached versions file and etag file
+    bundle :install, :artifice => "compact_index"
+
+    gemfile <<-G
+      source "#{source_uri}"
+      gem 'rack', '1.0.0'
+    G
+
+    versions = Pathname.new(Bundler.rubygems.user_home).join(
+      ".bundle", "cache", "compact_index",
+      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions"
+    )
+    # Modify the cached file. The ranged request will be based on this but,
+    # in this test, the range is ignored so this gets overwritten, allowing install.
+    versions.write "ruining this file"
+
+    bundle :install, :artifice => "compact_index_range_ignored"
+
+    expect(the_bundle).to include_gems "rack 1.0.0"
+  end
+
+  it "performs partial update with a non-empty range" do
+    build_repo4 do
+      build_gem "rack", "0.9.1"
+    end
+
+    # Initial install creates the cached versions file
+    install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+      source "#{source_uri}"
+      gem 'rack', '0.9.1'
+    G
+
+    update_repo4 do
+      build_gem "rack", "1.0.0"
+    end
+
+    install_gemfile <<-G, :artifice => "compact_index_partial_update", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+      source "#{source_uri}"
+      gem 'rack', '1.0.0'
+    G
 
     expect(the_bundle).to include_gems "rack 1.0.0"
   end
@@ -799,19 +860,43 @@ The checksum of /versions does not match the checksum provided by the server! So
       gem 'rack'
     G
 
-    # Create an empty file to trigger a partial download
-    versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index",
-      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions")
-    FileUtils.mkdir_p(File.dirname(versions))
-    FileUtils.touch(versions)
+    # Create a partial cache versions file
+    versions = Pathname.new(Bundler.rubygems.user_home).join(
+      ".bundle", "cache", "compact_index",
+      "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions"
+    )
+    versions.dirname.mkpath
+    versions.write("created_at")
 
     bundle :install, :artifice => "compact_index_concurrent_download"
 
-    expect(File.read(versions)).to start_with("created_at")
+    expect(versions.read).to start_with("created_at")
+    expect(the_bundle).to include_gems "rack 1.0.0"
+  end
+
+  it "performs a partial update that fails digest check, then a full update" do
+    build_repo4 do
+      build_gem "rack", "0.9.1"
+    end
+
+    install_gemfile <<-G, :artifice => "compact_index", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+      source "#{source_uri}"
+      gem 'rack', '0.9.1'
+    G
+
+    update_repo4 do
+      build_gem "rack", "1.0.0"
+    end
+
+    install_gemfile <<-G, :artifice => "compact_index_partial_update_bad_digest", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+      source "#{source_uri}"
+      gem 'rack', '1.0.0'
+    G
+
     expect(the_bundle).to include_gems "rack 1.0.0"
   end
 
-  it "performs full update if server endpoints serve partial content responses but don't have incremental content and provide no Etag" do
+  it "performs full update if server endpoints serve partial content responses but don't have incremental content and provide no digest" do
     build_repo4 do
       build_gem "rack", "0.9.1"
     end
@@ -825,7 +910,7 @@ The checksum of /versions does not match the checksum provided by the server! So
       build_gem "rack", "1.0.0"
     end
 
-    install_gemfile <<-G, :artifice => "compact_index_partial_update_no_etag_not_incremental", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+    install_gemfile <<-G, :artifice => "compact_index_partial_update_no_digest_not_incremental", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
       source "#{source_uri}"
       gem 'rack', '1.0.0'
     G
@@ -847,7 +932,7 @@ The checksum of /versions does not match the checksum provided by the server! So
     expected_rack_info_content = File.read(rake_info_path)
 
     # Modify the cache files. We expect them to be reset to the normal ones when we re-run :install
-    File.open(rake_info_path, "w") {|f| f << (expected_rack_info_content + "this is different") }
+    File.open(rake_info_path, "a") {|f| f << "this is different" }
 
     # Update the Gemfile so the next install does its normal things
     gemfile <<-G