Merge rubygems/bundler HEAD

Merge from 2af2520b4a

lib/bundler.rb

@@ -53,6 +53,7 @@ module Bundler
   autoload :GemHelpers,             File.expand_path("bundler/gem_helpers", __dir__)
   autoload :GemVersionPromoter,     File.expand_path("bundler/gem_version_promoter", __dir__)
   autoload :Graph,                  File.expand_path("bundler/graph", __dir__)
+  autoload :IncompleteSpecification, File.expand_path("bundler/incomplete_specification", __dir__)
   autoload :Index,                  File.expand_path("bundler/index", __dir__)
   autoload :Injector,               File.expand_path("bundler/injector", __dir__)
   autoload :Installer,              File.expand_path("bundler/installer", __dir__)
@@ -455,7 +456,7 @@ EOF
     end
 
     def local_platform
-      return Gem::Platform::RUBY if settings[:force_ruby_platform]
+      return Gem::Platform::RUBY if settings[:force_ruby_platform] || Gem.platforms == [Gem::Platform::RUBY]
       Gem::Platform.local
     end
 

lib/bundler/definition.rb

@@ -138,13 +138,13 @@ module Bundler
         @unlock[:gems] ||= @dependencies.map(&:name)
       else
         eager_unlock = expand_dependencies(@unlock[:gems] || [], true)
-        @unlock[:gems] = @locked_specs.for(eager_unlock, false, false).map(&:name)
+        @unlock[:gems] = @locked_specs.for(eager_unlock, false, platforms).map(&:name)
       end
 
       @dependency_changes = converge_dependencies
       @local_changes = converge_locals
 
-      @locked_specs_incomplete_for_platform = !@locked_specs.for(requested_dependencies & expand_dependencies(locked_dependencies), true, true)
+      @reresolve = nil
 
       @requires = compute_requires
     end
@@ -279,11 +279,8 @@ module Bundler
           end
         end
       else
-        last_resolve = converge_locked_specs
-        # Run a resolve against the locally available gems
         Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}")
-        expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, true)
+        @reresolve = reresolve
-        Resolver.resolve(expanded_dependencies, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms)
       end
     end
 
@@ -468,7 +465,7 @@ module Bundler
     private :sources
 
     def nothing_changed?
-      !@source_changes && !@dependency_changes && !@new_platform && !@path_changes && !@local_changes && !@locked_specs_incomplete_for_platform
+      !@source_changes && !@dependency_changes && !@new_platform && !@path_changes && !@local_changes
     end
 
     def unlocking?
@@ -477,8 +474,14 @@ module Bundler
 
     private
 
+    def reresolve
+      last_resolve = converge_locked_specs
+      expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, true)
+      Resolver.resolve(expanded_dependencies, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms)
+    end
+
     def filter_specs(specs, deps)
-      SpecSet.new(specs).for(expand_dependencies(deps, true), false, false)
+      SpecSet.new(specs).for(expand_dependencies(deps, true), false, platforms)
     end
 
     def materialize(dependencies)
@@ -502,6 +505,17 @@ module Bundler
         raise GemNotFound, "Could not find #{missing_specs_list.join(" nor ")}"
       end
 
+      if @reresolve.nil?
+        incomplete_specs = specs.incomplete_specs
+
+        if incomplete_specs.any?
+          Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies")
+          @unlock[:gems].concat(incomplete_specs.map(&:name))
+          @resolve = reresolve
+          specs = resolve.materialize(dependencies)
+        end
+      end
+
       unless specs["bundler"].any?
         bundler = sources.metadata_source.specs.search(Gem::Dependency.new("bundler", VERSION)).last
         specs["bundler"] = bundler
@@ -549,7 +563,6 @@ module Bundler
         [@new_platform, "you added a new platform to your gemfile"],
         [@path_changes, "the gemspecs for path gems changed"],
         [@local_changes, "the gemspecs for git local gems changed"],
-        [@locked_specs_incomplete_for_platform, "the lockfile does not have all gems needed for the current platform"],
       ].select(&:first).map(&:last).join(", ")
     end
 
@@ -725,7 +738,7 @@ module Bundler
             # if we won't need the source (according to the lockfile),
             # don't error if the path/git source isn't available
             next if specs.
-                    for(requested_dependencies, false, true).
+                    for(requested_dependencies, false).
                     none? {|locked_spec| locked_spec.source == s.source }
 
             raise

lib/bundler/dependency.rb

@@ -1,14 +1,11 @@
 # frozen_string_literal: true
 
 require "rubygems/dependency"
-require_relative "force_platform"
 require_relative "shared_helpers"
 require_relative "rubygems_ext"
 
 module Bundler
   class Dependency < Gem::Dependency
-    include ForcePlatform
-
     attr_reader :autorequire
     attr_reader :groups, :platforms, :gemfile, :git, :github, :branch, :ref, :force_ruby_platform
 
@@ -112,7 +109,7 @@ module Bundler
       @env            = options["env"]
       @should_include = options.fetch("should_include", true)
       @gemfile        = options["gemfile"]
-      @force_ruby_platform = options.fetch("force_ruby_platform", default_force_ruby_platform)
+      @force_ruby_platform = options["force_ruby_platform"]
 
       @autorequire = Array(options["require"] || []) if options.key?("require")
     end

lib/bundler/force_platform.rb

@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-module Bundler
-  module ForcePlatform
-    private
-
-    # The `:force_ruby_platform` value used by dependencies for resolution, and
-    # by locked specifications for materialization is `false` by default, except
-    # for TruffleRuby. TruffleRuby generally needs to force the RUBY platform
-    # variant unless the name is explicitly allowlisted.
-
-    def default_force_ruby_platform
-      return false unless Bundler.current_ruby.truffleruby?
-
-      !Gem::Platform::REUSE_AS_BINARY_ON_TRUFFLERUBY.include?(name)
-    end
-  end
-end

lib/bundler/incomplete_specification.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Bundler
+  class IncompleteSpecification
+    attr_reader :name, :platform
+
+    def initialize(name, platform)
+      @name = name
+      @platform = platform
+    end
+  end
+end

lib/bundler/lazy_specification.rb

@@ -1,16 +1,13 @@
 # frozen_string_literal: true
 
-require_relative "force_platform"
 require_relative "match_platform"
 
 module Bundler
   class LazySpecification
-    include ForcePlatform
     include MatchPlatform
 
     attr_reader :name, :version, :dependencies, :platform
-    attr_writer :force_ruby_platform
+    attr_accessor :source, :remote, :force_ruby_platform
-    attr_accessor :source, :remote
 
     def initialize(name, version, platform, source = nil)
       @name          = name
@@ -19,23 +16,16 @@ module Bundler
       @platform      = platform || Gem::Platform::RUBY
       @source        = source
       @specification = nil
-      @force_ruby_platform = nil
     end
 
     def full_name
-      if platform == Gem::Platform::RUBY || platform.nil?
+      if platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
         "#{@name}-#{@version}-#{platform}"
       end
     end
 
-    def force_ruby_platform
-      return @force_ruby_platform unless @force_ruby_platform.nil?
-
-      default_force_ruby_platform
-    end
-
     def ==(other)
       identifier == other.identifier
     end
@@ -71,7 +61,7 @@ module Bundler
     def to_lock
       out = String.new
 
-      if platform == Gem::Platform::RUBY || platform.nil?
+      if platform == Gem::Platform::RUBY
         out << "    #{name} (#{version})\n"
       else
         out << "    #{name} (#{version}-#{platform})\n"
@@ -85,7 +75,17 @@ module Bundler
       out
     end
 
-    def __materialize__
+    def materialize_for_installation
+      __materialize__(ruby_platform_materializes_to_ruby_platform? ? platform : Bundler.local_platform)
+    end
+
+    def materialize_for_resolution
+      return self unless Gem::Platform.match_spec?(self)
+
+      __materialize__(platform)
+    end
+
+    def __materialize__(platform)
       @specification = if source.is_a?(Source::Gemspec) && source.gemspec.name == name
         source.gemspec.tap {|s| s.source = source }
       else
@@ -94,10 +94,9 @@ module Bundler
         else
           ruby_platform_materializes_to_ruby_platform? ? self : Dependency.new(name, version)
         end
-        platform_object = ruby_platform_materializes_to_ruby_platform? ? Gem::Platform.new(platform) : Gem::Platform.local
         candidates = source.specs.search(search_object)
         same_platform_candidates = candidates.select do |spec|
-          MatchPlatform.platforms_match?(spec.platform, platform_object)
+          MatchPlatform.platforms_match?(spec.platform, platform)
         end
         installable_candidates = same_platform_candidates.select do |spec|
           spec.is_a?(StubSpecification) ||
@@ -115,7 +114,7 @@ module Bundler
     end
 
     def to_s
-      @__to_s ||= if platform == Gem::Platform::RUBY || platform.nil?
+      @__to_s ||= if platform == Gem::Platform::RUBY
         "#{name} (#{version})"
       else
         "#{name} (#{version}-#{platform})"

lib/bundler/remote_specification.rb

@@ -16,7 +16,8 @@ module Bundler
     def initialize(name, version, platform, spec_fetcher)
       @name         = name
       @version      = Gem::Version.create version
-      @platform     = platform
+      @original_platform = platform || Gem::Platform::RUBY
+      @platform     = Gem::Platform.new(platform)
       @spec_fetcher = spec_fetcher
       @dependencies = nil
     end
@@ -35,10 +36,10 @@ module Bundler
     end
 
     def full_name
-      if platform == Gem::Platform::RUBY || platform.nil?
+      if @original_platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
       else
-        "#{@name}-#{@version}-#{platform}"
+        "#{@name}-#{@version}-#{@original_platform}"
       end
     end
 
@@ -105,7 +106,7 @@ module Bundler
     end
 
     def _remote_specification
-      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @platform])
+      @_remote_specification ||= @spec_fetcher.fetch_spec([@name, @version, @original_platform])
       @_remote_specification || raise(GemspecError, "Gemspec data for #{full_name} was" \
         " missing from the server! Try installing with `--full-index` as a workaround.")
     end

lib/bundler/resolver.rb

@@ -22,17 +22,16 @@ module Bundler
       metadata_requirements, regular_requirements = requirements.partition {|dep| dep.name.end_with?("\0") }
       resolver = new(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms, metadata_requirements)
       result = resolver.start(requirements)
-      SpecSet.new(SpecSet.new(result).for(regular_requirements))
+      SpecSet.new(SpecSet.new(result).for(regular_requirements, false, platforms))
     end
 
     def initialize(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms, metadata_requirements)
       @source_requirements = source_requirements
       @metadata_requirements = metadata_requirements
-      @base = base
       @resolver = Molinillo::Resolver.new(self, self)
       @search_for = {}
       @base_dg = Molinillo::DependencyGraph.new
-      @base.each do |ls|
+      @base = base.materialized_for_resolution do |ls|
         dep = Dependency.new(ls.name, ls.version)
         @base_dg.add_vertex(ls.name, DepProxy.get_proxy(dep, ls.platform), true)
       end

lib/bundler/rubygems_ext.rb

@@ -222,9 +222,27 @@ module Gem
     MINGW = Gem::Platform.new("x86-mingw32")
     X64_MINGW = [Gem::Platform.new("x64-mingw32"),
                  Gem::Platform.new("x64-mingw-ucrt")].freeze
+  end
 
-    if RUBY_ENGINE == "truffleruby" && !defined?(REUSE_AS_BINARY_ON_TRUFFLERUBY)
+  Platform.singleton_class.module_eval do
-      REUSE_AS_BINARY_ON_TRUFFLERUBY = %w[libv8 sorbet-static].freeze
+    unless Platform.singleton_methods.include?(:match_spec?)
+      def match_spec?(spec)
+        match_gem?(spec.platform, spec.name)
+      end
+
+      def match_gem?(platform, gem_name)
+        match_platforms?(platform, Gem.platforms)
+      end
+
+      private
+
+      def match_platforms?(platform, platforms)
+        platforms.any? do |local_platform|
+          platform.nil? ||
+            local_platform == platform ||
+            (local_platform != Gem::Platform::RUBY && local_platform =~ platform)
+        end
+      end
     end
   end
 

lib/bundler/spec_set.rb

@@ -11,30 +11,27 @@ module Bundler
       @specs = specs
     end
 
-    def for(dependencies, check = false, match_current_platform = false)
+    def for(dependencies, check = false, platforms = [nil])
-      # dep.name => [list, of, deps]
+      handled = ["bundler"].product(platforms).map {|k| [k, true] }.to_h
-      handled = Hash.new {|h, k| h[k] = [] }
+      deps = dependencies.product(platforms).map {|dep, platform| [dep.name, platform && dep.force_ruby_platform ? Gem::Platform::RUBY : platform] }
-      deps = dependencies.dup
       specs = []
 
       loop do
         break unless dep = deps.shift
-        next if handled[dep.name].any? {|d| match_current_platform || d.__platform == dep.__platform } || dep.name == "bundler"
+        next if handled.key?(dep)
 
-        # use a hash here to ensure constant lookup time in the `any?` call above
+        handled[dep] = true
-        handled[dep.name] << dep
 
-        specs_for_dep = specs_for_dependency(dep, match_current_platform)
+        specs_for_dep = specs_for_dependency(*dep)
         if specs_for_dep.any?
           specs.concat(specs_for_dep)
 
           specs_for_dep.first.dependencies.each do |d|
             next if d.type == :development
-            d = DepProxy.get_proxy(Dependency.new(d.name, d.requirement), dep.__platform) unless match_current_platform
+            deps << [d.name, dep[1]]
-            deps << d
           end
         elsif check
-          return false
+          specs << IncompleteSpecification.new(*dep)
         end
       end
 
@@ -42,9 +39,7 @@ module Bundler
         specs << spec
       end
 
-      specs.uniq! unless match_current_platform
+      specs
-
-      check ? true : specs
     end
 
     def [](key)
@@ -71,12 +66,12 @@ module Bundler
     end
 
     def materialize(deps)
-      materialized = self.for(deps, false, true)
+      materialized = self.for(deps, true).uniq
 
       materialized.map! do |s|
         next s unless s.is_a?(LazySpecification)
         s.source.local!
-        s.__materialize__ || s
+        s.materialize_for_installation || s
       end
       SpecSet.new(materialized)
     end
@@ -89,16 +84,29 @@ module Bundler
         next s unless s.is_a?(LazySpecification)
         s.source.local!
         s.source.remote!
-        spec = s.__materialize__
+        spec = s.materialize_for_installation
         raise GemNotFound, "Could not find #{s.full_name} in any of the sources" unless spec
         spec
       end
     end
 
+    def materialized_for_resolution
+      materialized = @specs.map do |s|
+        spec = s.materialize_for_resolution
+        yield spec if spec
+        spec
+      end.compact
+      SpecSet.new(materialized)
+    end
+
     def missing_specs
       @specs.select {|s| s.is_a?(LazySpecification) }
     end
 
+    def incomplete_specs
+      @specs.select {|s| s.is_a?(IncompleteSpecification) }
+    end
+
     def merge(set)
       arr = sorted.dup
       set.each do |set_spec|
@@ -173,12 +181,12 @@ module Bundler
       @specs.sort_by(&:name).each {|s| yield s }
     end
 
-    def specs_for_dependency(dep, match_current_platform)
+    def specs_for_dependency(name, platform)
-      specs_for_name = lookup[dep.name]
+      specs_for_name = lookup[name]
-      if match_current_platform
+      if platform.nil?
-        GemHelpers.select_best_platform_match(specs_for_name, Bundler.local_platform)
+        GemHelpers.select_best_platform_match(specs_for_name.select {|s| Gem::Platform.match_spec?(s) }, Bundler.local_platform)
       else
-        specs_for_name_and_platform = GemHelpers.select_best_platform_match(specs_for_name, dep.force_ruby_platform ? Gem::Platform::RUBY : dep.__platform)
+        specs_for_name_and_platform = GemHelpers.select_best_platform_match(specs_for_name, platform)
         specs_for_name_and_platform.any? ? specs_for_name_and_platform : specs_for_name
       end
     end

spec/bundler/install/gemfile/sources_spec.rb

@@ -1563,4 +1563,43 @@ RSpec.describe "bundle install with gems on multiple sources" do
       L
     end
   end
+
+  context "when mistakenly adding a top level gem already depended on and cached under the wrong source" do
+    before do
+      build_repo4 do
+        build_gem "some_private_gem", "0.1.0" do |s|
+          s.add_dependency "example", "~> 1.0"
+        end
+      end
+
+      build_repo2 do
+        build_gem "example", "1.0.0"
+      end
+
+      install_gemfile <<~G, :artifice => "compact_index"
+        source "https://gem.repo2"
+
+        source "https://gem.repo4" do
+          gem "some_private_gem"
+        end
+      G
+
+      gemfile <<~G
+        source "https://gem.repo2"
+
+        source "https://gem.repo4" do
+          gem "some_private_gem"
+          gem "example" # MISTAKE, example is not available at gem.repo4
+        end
+      G
+    end
+
+    it "shows a proper error message and does not generate a corrupted lockfile" do
+      expect do
+        bundle :install, :artifice => "compact_index", :raise_on_error => false, :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo4.to_s }
+      end.not_to change { lockfile }
+
+      expect(err).to include("Could not find gem 'example' in rubygems repository https://gem.repo4/")
+    end
+  end
 end

spec/bundler/install/yanked_spec.rb

@@ -15,7 +15,7 @@ RSpec.context "when installing a bundle that includes yanked gems" do
            foo (10.0.0)
 
        PLATFORMS
-         ruby
+         #{lockfile_platforms}
 
        DEPENDENCIES
          foo (= 10.0.0)
@@ -57,7 +57,7 @@ RSpec.context "when using gem before installing" do
           rack (0.9.1)
 
       PLATFORMS
-        ruby
+        #{lockfile_platforms}
 
       DEPENDENCIES
         rack (= 0.9.1)
@@ -86,7 +86,7 @@ RSpec.context "when using gem before installing" do
           rack_middleware (1.0)
 
       PLATFORMS
-        ruby
+        #{lockfile_platforms}
 
       DEPENDENCIES
         rack (= 0.9.1)

spec/bundler/resolver/basic_spec.rb

@@ -233,7 +233,7 @@ Bundler could not find compatible versions for gem "a":
     it "resolves foo only to latest patch - changing dependency declared case" do
       # bar is locked AND a declared dependency in the Gemfile, so it will not move, and therefore
       # foo can only move up to 1.4.4.
-      @base << build_spec("bar", "2.0.3").first
+      @base << Bundler::LazySpecification.new("bar", "2.0.3", nil)
       should_conservative_resolve_and_include :patch, ["foo"], %w[foo-1.4.4 bar-2.0.3]
     end
 

spec/bundler/runtime/platform_spec.rb

@@ -281,30 +281,59 @@ RSpec.describe "Bundler.setup with multi platform stuff" do
 
     expect(the_bundle).to include_gems "platform_specific 1.0 RUBY"
 
-    build_repo4 do
+    simulate_platform "x86_64-linux" do
-      build_gem "libv8"
+      build_repo4 do
+        build_gem "libv8"
 
-      build_gem "libv8" do |s|
+        build_gem "libv8" do |s|
-        s.platform = Bundler.local_platform
+          s.platform = "x86_64-linux"
+        end
       end
-    end
 
+      gemfile <<-G
+        source "#{file_uri_for(gem_repo4)}"
+        gem "libv8"
+      G
+
+      lockfile <<-L
+        GEM
+          remote: #{file_uri_for(gem_repo4)}/
+          specs:
+            libv8 (1.0)
+
+        PLATFORMS
+          ruby
+
+        DEPENDENCIES
+          libv8
+
+        BUNDLED WITH
+           #{Bundler::VERSION}
+      L
+
+      bundle "install"
+
+      expect(the_bundle).to include_gems "libv8 1.0 x86_64-linux"
+    end
+  end
+
+  it "doesn't pull platform specific gems on truffleruby, even if lockfile only includes those", :truffleruby_only do
     gemfile <<-G
-      source "#{file_uri_for(gem_repo4)}"
+      source "#{file_uri_for(gem_repo1)}"
-      gem "libv8"
+      gem "platform_specific"
     G
 
     lockfile <<-L
       GEM
-        remote: #{file_uri_for(gem_repo4)}/
+        remote: #{file_uri_for(gem_repo1)}/
         specs:
-          libv8 (1.0)
+          platform_specific (1.0-x86-darwin-100)
 
       PLATFORMS
-        ruby
+        x86-darwin-100
 
       DEPENDENCIES
-        libv8
+        platform_specific
 
       BUNDLED WITH
          #{Bundler::VERSION}
@@ -312,7 +341,7 @@ RSpec.describe "Bundler.setup with multi platform stuff" do
 
     bundle "install"
 
-    expect(the_bundle).to include_gems "libv8 1.0 #{Bundler.local_platform}"
+    expect(the_bundle).to include_gems "platform_specific 1.0 RUBY"
   end
 
   it "allows specifying only-ruby-platform on windows with dependency platforms" do
@@ -379,6 +408,7 @@ RSpec.describe "Bundler.setup with multi platform stuff" do
         gem "requires_platform_specific"
       G
 
+      expect(out).to include("lockfile does not have all gems needed for the current platform")
       expect(the_bundle).to include_gem "platform_specific 1.0 x64-mingw32"
     end
   end

spec/bundler/runtime/setup_spec.rb

@@ -634,6 +634,7 @@ RSpec.describe "Bundler.setup" do
 
       ruby "require '#{system_gem_path("gems/bundler-9.99.9.beta1/lib/bundler.rb")}'; Bundler.setup", :env => { "DEBUG" => "1" }
       expect(out).to include("Found no changes, using resolution from the lockfile")
+      expect(out).not_to include("lockfile does not have all gems needed for the current platform")
       expect(err).to be_empty
     end
 

spec/bundler/support/builders.rb

@@ -94,8 +94,8 @@ module Spec
         end
 
         build_gem "platform_specific" do |s|
-          s.platform = Bundler.local_platform
+          s.platform = Gem::Platform.local
-          s.write "lib/platform_specific.rb", "PLATFORM_SPECIFIC = '1.0.0 #{Bundler.local_platform}'"
+          s.write "lib/platform_specific.rb", "PLATFORM_SPECIFIC = '1.0.0 #{Gem::Platform.local}'"
         end
 
         build_gem "platform_specific" do |s|

spec/bundler/support/indexes.rb

@@ -26,6 +26,7 @@ module Spec
         end
       end
       args[0] ||= [] # base
+      args[0].each {|ls| ls.source = default_source }
       args[1] ||= Bundler::GemVersionPromoter.new # gem_version_promoter
       args[2] ||= [] # additional_base_requirements
       args[3] ||= @platforms # platforms