[rubygems/rubygems] Use the server checksum, then calculate from gem on disk if possible

1. Use the checksum provided by the server if provided: provides security
knowing if the gem you downloaded matches the gem on the server

2. Calculate the checksum from the gem on disk: provides security knowing
if the gem has changed between installs

3. In some cases, neither is possible in which case we don't put anything
in the checksum and we maintain functionality as it is today

Add the checksums to specs in the index if we already have them

Prior to checksums, we didn't lose any information when overwriting specs
in the index with stubs. But now when we overwrite EndpointSpecifications
or RemoteSpecifications with more generic specs, we could lose checksum
info. This manually sets checksum info so we keep it in the index.

https://github.com/rubygems/rubygems/commit/de00a4f153

11 files changed, 152 insertions, 80 deletions

diff --git a/lib/bundler/checksum.rb b/lib/bundler/checksum.rb
index 2e0a80cac2..0b618d5033 100644
--- a/lib/bundler/checksum.rb
+++ b/lib/bundler/checksum.rb
@@ -2,22 +2,37 @@
 
 module Bundler
   class Checksum
-    attr_reader :name, :version, :platform
-    attr_accessor :checksum
+    attr_reader :name, :version, :platform, :checksums
 
-    SHA256 = /\Asha256-([a-z0-9]{64}|[A-Za-z0-9+\/=]{44})\z/.freeze
+    SHA256 = %r{\Asha256-([a-z0-9]{64}|[A-Za-z0-9+\/=]{44})\z}.freeze
 
-    def initialize(name, version, platform, checksum = nil)
+    def initialize(name, version, platform, checksums = [])
       @name     = name
       @version  = version
       @platform = platform || Gem::Platform::RUBY
-      @checksum = checksum
+      @checksums = checksums
 
-      if @checksum && @checksum !~ SHA256
-        raise ArgumentError, "invalid checksum (#{@checksum})"
+      # can expand this validation when we support more hashing algos later
+      if @checksums.any? && @checksums.all? {|c| c !~ SHA256 }
+        raise ArgumentError, "invalid checksums (#{@checksums})"
       end
     end
 
+    def self.digest_from_file_source(file_source)
+      raise ArgumentError, "not a valid file source: #{file_source}" unless file_source.respond_to?(:with_read_io)
+
+      file_source.with_read_io do |io|
+        digest = Bundler::SharedHelpers.digest(:SHA256).new
+        digest << io.read(16_384) until io.eof?
+        io.rewind
+        digest
+      end
+    end
+
+    def full_name
+      GemHelpers.spec_full_name(@name, @version, @platform)
+    end
+
     def match_spec?(spec)
       name == spec.name &&
         version == spec.version &&
@@ -26,17 +41,17 @@ module Bundler
 
     def to_lock
       out = String.new
-
-      if platform == Gem::Platform::RUBY
-        out << "  #{name} (#{version})"
-      else
-        out << "  #{name} (#{version}-#{platform})"
-      end
-
-      out << " #{checksum}" if checksum
+      out << "  #{GemHelpers.lock_name(name, version, platform)}"
+      out << " #{sha256}" if sha256
       out << "\n"
 
       out
     end
+
+    private
+
+    def sha256
+      @checksums.find {|c| c =~ SHA256 }
+    end
   end
 end
diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 6b066051d8..14f6746331 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -114,7 +114,7 @@ module Bundler
         @originally_locked_specs = @locked_specs
         @locked_sources = []
         @locked_platforms = []
-        @locked_checksums = []
+        @locked_checksums = {}
       end
 
       locked_gem_sources = @locked_sources.select {|s| s.is_a?(Source::Rubygems) }
diff --git a/lib/bundler/endpoint_specification.rb b/lib/bundler/endpoint_specification.rb
index 4c41285043..863544b1f9 100644
--- a/lib/bundler/endpoint_specification.rb
+++ b/lib/bundler/endpoint_specification.rb
@@ -104,11 +104,6 @@ module Bundler
       @remote_specification = spec
     end
 
-    def to_checksum
-      digest = "sha256-#{checksum}" if checksum
-      Bundler::Checksum.new(name, version, platform, digest)
-    end
-
     private
 
     def _remote_specification
diff --git a/lib/bundler/gem_helpers.rb b/lib/bundler/gem_helpers.rb
index 2e6d788f9c..ed39511a10 100644
--- a/lib/bundler/gem_helpers.rb
+++ b/lib/bundler/gem_helpers.rb
@@ -113,5 +113,23 @@ module Bundler
       same_runtime_deps && same_metadata_deps
     end
     module_function :same_deps
+
+    def spec_full_name(name, version, platform)
+      if platform == Gem::Platform::RUBY
+        "#{name}-#{version}"
+      else
+        "#{name}-#{version}-#{platform}"
+      end
+    end
+    module_function :spec_full_name
+
+    def lock_name(name, version, platform)
+      if platform == Gem::Platform::RUBY
+        "#{name} (#{version})"
+      else
+        "#{name} (#{version}-#{platform})"
+      end
+    end
+    module_function :lock_name
   end
 end
diff --git a/lib/bundler/lazy_specification.rb b/lib/bundler/lazy_specification.rb
index b4aadb0b5c..a17c8b90e5 100644
--- a/lib/bundler/lazy_specification.rb
+++ b/lib/bundler/lazy_specification.rb
@@ -20,11 +20,7 @@ module Bundler
     end
 
     def full_name
-      @full_name ||= if platform == Gem::Platform::RUBY
-        "#{@name}-#{@version}"
-      else
-        "#{@name}-#{@version}-#{platform}"
-      end
+      @full_name ||= GemHelpers.spec_full_name(@name, @version, platform)
     end
 
     def ==(other)
@@ -61,12 +57,7 @@ module Bundler
 
     def to_lock
       out = String.new
-
-      if platform == Gem::Platform::RUBY
-        out << "    #{name} (#{version})\n"
-      else
-        out << "    #{name} (#{version}-#{platform})\n"
-      end
+      out << "    #{GemHelpers.lock_name(name, version, platform)}\n"
 
       dependencies.sort_by(&:to_s).uniq.each do |dep|
         next if dep.type == :development
@@ -76,17 +67,18 @@ module Bundler
       out
     end
 
-    #def materialize_for_checksum
-      #if @specification
-        #yield
-      #else
-        #materialize_for_installation
-
-        #yield
+    def materialize_for_checksum(&blk)
+      #
+      # See comment about #ruby_platform_materializes_to_ruby_platform?
+      # If the old lockfile format is present where there is no specific
+      # platform, then we should skip locking checksums as it is not
+      # deterministic which platform variant is locked.
+      #
+      return unless ruby_platform_materializes_to_ruby_platform?
 
-        #@specification = nil
-      #end
-    #end
+      s = materialize_for_installation
+      yield s if block_given?
+    end
 
     def materialize_for_installation
       source.local!
@@ -134,11 +126,7 @@ module Bundler
     end
 
     def to_s
-      @to_s ||= if platform == Gem::Platform::RUBY
-        "#{name} (#{version})"
-      else
-        "#{name} (#{version}-#{platform})"
-      end
+      @__to_s ||= GemHelpers.lock_name(name, version, platform)
     end
 
     def git_version
@@ -146,20 +134,6 @@ module Bundler
       " #{source.revision[0..6]}"
     end
 
-    def to_checksum
-      return nil unless @specification
-
-      #
-      # See comment about #ruby_platform_materializes_to_ruby_platform?
-      # If the old lockfile format is present where there is no specific
-      # platform, then we should skip locking checksums as it is not
-      # deterministic which platform variant is locked.
-      #
-      return nil unless ruby_platform_materializes_to_ruby_platform?
-
-      @specification.to_checksum
-    end
-
     private
 
     def use_exact_resolved_specifications?
diff --git a/lib/bundler/lockfile_generator.rb b/lib/bundler/lockfile_generator.rb
index 11e8e3f103..52b3b411aa 100644
--- a/lib/bundler/lockfile_generator.rb
+++ b/lib/bundler/lockfile_generator.rb
@@ -68,17 +68,14 @@ module Bundler
 
     def add_checksums
       out << "\nCHECKSUMS\n"
-
       definition.resolve.sort_by(&:full_name).each do |spec|
         checksum = spec.to_checksum if spec.respond_to?(:to_checksum)
-
-        #if spec.is_a?(LazySpecification)
-          #spec.materialize_for_checksum do
-            #checksum ||= spec.to_checksum if spec.respond_to?(:to_checksum)
-          #end
-        #end
-
-        checksum ||= definition.locked_checksums.find {|c| c.match_spec?(spec) }
+        if spec.is_a?(LazySpecification)
+          spec.materialize_for_checksum do |materialized_spec|
+            checksum ||= materialized_spec.to_checksum if materialized_spec&.respond_to?(:to_checksum)
+          end
+        end
+        checksum ||= definition.locked_checksums[spec.full_name]
 
         out << checksum.to_lock if checksum
       end
diff --git a/lib/bundler/lockfile_parser.rb b/lib/bundler/lockfile_parser.rb
index fc331a928c..001de06d53 100644
--- a/lib/bundler/lockfile_parser.rb
+++ b/lib/bundler/lockfile_parser.rb
@@ -66,7 +66,7 @@ module Bundler
       @sources      = []
       @dependencies = {}
       @parse_method = nil
-      @checksums    = []
+      @checksums    = {}
       @specs        = {}
 
       if lockfile.match?(/<<<<<<<|=======|>>>>>>>|\|\|\|\|\|\|\|/)
@@ -193,7 +193,8 @@ module Bundler
 
         version = Gem::Version.new(version)
         platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY
-        @checksums << Bundler::Checksum.new(name, version, platform, checksum)
+        checksum = Bundler::Checksum.new(name, version, platform, [checksum])
+        @checksums[checksum.full_name] = checksum
       end
     end
 
diff --git a/lib/bundler/remote_specification.rb b/lib/bundler/remote_specification.rb
index f626a3218e..e8054dbbd5 100644
--- a/lib/bundler/remote_specification.rb
+++ b/lib/bundler/remote_specification.rb
@@ -93,12 +93,56 @@ module Bundler
       " #{source.revision[0..6]}"
     end
 
+    # we don't get the checksum from a server like we could with EndpointSpecs
+    # calculating the checksum from the file on disk still provides some measure of security
+    # if it changes from install to install, that is cause for concern
+    def to_checksum
+      @checksum ||= begin
+        gem_path = fetch_gem
+        require "rubygems/package"
+        package = Gem::Package.new(gem_path)
+        digest = Bundler::Checksum.digest_from_file_source(package.gem)
+        digest.hexdigest!
+      end
+
+      digest = "sha256-#{@checksum}" if @checksum
+      Bundler::Checksum.new(name, version, platform, [digest])
+    end
+
     private
 
     def to_ary
       nil
     end
 
+    def fetch_gem
+      fetch_platform
+
+      cache_path = download_cache_path || default_cache_path_for_rubygems_dir
+      gem_path = "#{cache_path}/#{file_name}"
+      return gem_path if File.exist?(gem_path)
+
+      SharedHelpers.filesystem_access(cache_path) do |p|
+        FileUtils.mkdir_p(p)
+      end
+
+      Bundler.rubygems.download_gem(self, remote.uri, cache_path)
+
+      gem_path
+    end
+
+    def download_cache_path
+      return unless Bundler.feature_flag.global_gem_cache?
+      return unless remote
+      return unless remote.cache_slug
+
+      Bundler.user_cache.join("gems", remote.cache_slug)
+    end
+
+    def default_cache_path_for_rubygems_dir
+      "#{Bundler.bundle_path}/cache"
+    end
+
     def _remote_specification
       @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @original_platform])
       @_remote_specification || raise(GemspecError, "Gemspec data for #{full_name} was" \
diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb
index 38035a00ac..22e3185b7f 100644
--- a/lib/bundler/rubygems_gem_installer.rb
+++ b/lib/bundler/rubygems_gem_installer.rb
@@ -120,13 +120,10 @@ module Bundler
       return true unless checksum
       return true unless source = @package.instance_variable_get(:@gem)
       return true unless source.respond_to?(:with_read_io)
-      digest = source.with_read_io do |io|
-        digest = SharedHelpers.digest(:SHA256).new
-        digest << io.read(16_384) until io.eof?
-        io.rewind
-        send(checksum_type(checksum), digest)
-      end
-      unless digest == checksum
+      digest = Bundler::Checksum.digest_from_file_source(source)
+      calculated_checksum = send(checksum_type(checksum), digest)
+
+      unless calculated_checksum == checksum
         raise SecurityError, <<-MESSAGE
           Bundler cannot continue installing #{spec.name} (#{spec.version}).
           The checksum for the downloaded `#{spec.full_name}.gem` does not match \
@@ -143,7 +140,7 @@ module Bundler
           2. run `bundle install`
 
           (More info: The expected SHA256 checksum was #{checksum.inspect}, but the \
-          checksum for the downloaded gem was #{digest.inspect}.)
+          checksum for the downloaded gem was #{calculated_checksum.inspect}.)
           MESSAGE
       end
       true
diff --git a/lib/bundler/stub_specification.rb b/lib/bundler/stub_specification.rb
index 6f4264e561..0ce68b964c 100644
--- a/lib/bundler/stub_specification.rb
+++ b/lib/bundler/stub_specification.rb
@@ -9,6 +9,7 @@ module Bundler
       spec
     end
 
+    attr_reader :checksum
     attr_accessor :stub, :ignored
 
     def source=(source)
@@ -92,6 +93,16 @@ module Bundler
       stub.raw_require_paths
     end
 
+    def add_checksum(checksum)
+      @checksum ||= checksum
+    end
+
+    def to_checksum
+      return Bundler::Checksum.new(name, version, platform, ["sha256-#{checksum}"]) if checksum
+
+      _remote_specification&.to_checksum
+    end
+
     private
 
     def _remote_specification
diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb
index 6f69ee22ce..8af62cced7 100644
--- a/lib/rubygems/specification.rb
+++ b/lib/rubygems/specification.rb
@@ -761,6 +761,8 @@ class Gem::Specification < Gem::BasicSpecification
 
   attr_accessor :specification_version
 
+  attr_reader :checksum
+
   def self._all # :nodoc:
     @@all ||= Gem.loaded_specs.values | stubs.map(&:to_spec)
   end
@@ -2738,4 +2740,22 @@ class Gem::Specification < Gem::BasicSpecification
   def raw_require_paths # :nodoc:
     @require_paths
   end
+
+  def add_checksum(checksum)
+    @checksum ||= checksum
+  end
+
+  # if we don't get the checksum from the server
+  # calculating the checksum from the file on disk still provides some measure of security
+  # if it changes from install to install, that is cause for concern
+  def to_checksum
+    return Bundler::Checksum.new(name, version, platform, ["sha256-#{checksum}"]) if checksum
+    return Bundler::Checksum.new(name, version, platform) unless File.exist?(cache_file)
+
+    require "rubygems/package"
+    package = Gem::Package.new(cache_file)
+    digest = Bundler::Checksum.digest_from_file_source(package.gem)
+    calculated_checksum = digest.hexdigest!
+    Bundler::Checksum.new(name, version, platform, ["sha256-#{calculated_checksum}"]) if calculated_checksum
+  end
 end