Merge pull request #4110 from rubygems/compact_index_api

Respect `required_ruby_version` and `required_rubygems_version` constraints when looking for `gem install` candidates

(cherry picked from commit c533e9f)

.github/workflows/install-rubygems.yml

@@ -62,6 +62,8 @@ jobs:
       - name: Check bundler install didn't hit the network
         run: if grep -q 'GET http' output.txt; then false; else true; fi
         working-directory: ./bundler
+      - name: Check rails can be installed
+        run: gem install rails
     timeout-minutes: 10
 
   install_rubygems_windows:

Manifest.txt

@@ -52,6 +52,7 @@ bundler/lib/bundler/cli/update.rb
 bundler/lib/bundler/cli/viz.rb
 bundler/lib/bundler/compact_index_client.rb
 bundler/lib/bundler/compact_index_client/cache.rb
+bundler/lib/bundler/compact_index_client/gem_parser.rb
 bundler/lib/bundler/compact_index_client/updater.rb
 bundler/lib/bundler/constants.rb
 bundler/lib/bundler/current_ruby.rb
@@ -412,6 +413,7 @@ lib/rubygems/requirement.rb
 lib/rubygems/resolver.rb
 lib/rubygems/resolver/activation_request.rb
 lib/rubygems/resolver/api_set.rb
+lib/rubygems/resolver/api_set/gem_parser.rb
 lib/rubygems/resolver/api_specification.rb
 lib/rubygems/resolver/best_set.rb
 lib/rubygems/resolver/composed_set.rb

bundler/lib/bundler/compact_index_client/cache.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "gem_parser"
+
 module Bundler
   class CompactIndexClient
     class Cache
@@ -92,19 +94,9 @@ def lines(path)
         header ? lines[header + 1..-1] : lines
       end
 
-      def parse_gem(string)
-        version_and_platform, rest = string.split(" ", 2)
-        version, platform = version_and_platform.split("-", 2)
-        dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
-        dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
-        requirements = requirements ? requirements.map {|r| parse_dependency(r) } : []
-        [version, platform, dependencies, requirements]
-      end
-
-      def parse_dependency(string)
-        dependency = string.split(":")
-        dependency[-1] = dependency[-1].split("&") if dependency.size > 1
-        dependency
+      def parse_gem(line)
+        @dependency_parser ||= GemParser.new
+        @dependency_parser.parse(line)
       end
 
       def info_roots

bundler/lib/bundler/compact_index_client/gem_parser.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Bundler
+  class CompactIndexClient
+    if defined?(Gem::Resolver::APISet::GemParser)
+      GemParser = Gem::Resolver::APISet::GemParser
+    else
+      class GemParser
+        def parse(line)
+          version_and_platform, rest = line.split(" ", 2)
+          version, platform = version_and_platform.split("-", 2)
+          dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+          dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+          requirements = requirements ? requirements.map {|d| parse_dependency(d) } : []
+          [version, platform, dependencies, requirements]
+        end
+
+        private
+
+        def parse_dependency(string)
+          dependency = string.split(":")
+          dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+          dependency
+        end
+      end
+    end
+  end
+end

bundler/spec/support/artifice/vcr.rb

@@ -39,7 +39,7 @@ def recorded_response
         response_io = ::Net::BufferedIO.new(response_file)
         ::Net::HTTPResponse.read_new(response_io).tap do |response|
           response.decode_content = request.decode_content if request.respond_to?(:decode_content)
-          response.uri = request.uri if request.respond_to?(:uri)
+          response.uri = request.uri
 
           response.reading_body(response_io, request.response_body_permitted?) do
             response_block.call(response) if response_block

lib/rubygems/dependency_installer.rb

@@ -286,6 +286,7 @@ def resolve_dependencies(dep_or_name, version) # :nodoc:
 
     installer_set = Gem::Resolver::InstallerSet.new @domain
     installer_set.ignore_installed = (@minimal_deps == false) || @only_install_dir
+    installer_set.force = @force
 
     if consider_local?
       if dep_or_name =~ /\.gem$/ and File.file? dep_or_name

lib/rubygems/installer.rb

@@ -638,27 +638,6 @@ def ensure_loadable_spec
     end
   end
 
-  def ensure_required_ruby_version_met # :nodoc:
-    if rrv = spec.required_ruby_version
-      ruby_version = Gem.ruby_version
-      unless rrv.satisfied_by? ruby_version
-        raise Gem::RuntimeRequirementNotMetError,
-          "#{spec.name} requires Ruby version #{rrv}. The current ruby version is #{ruby_version}."
-      end
-    end
-  end
-
-  def ensure_required_rubygems_version_met # :nodoc:
-    if rrgv = spec.required_rubygems_version
-      unless rrgv.satisfied_by? Gem.rubygems_version
-        rg_version = Gem::VERSION
-        raise Gem::RuntimeRequirementNotMetError,
-          "#{spec.name} requires RubyGems version #{rrgv}. The current RubyGems version is #{rg_version}. " +
-          "Try 'gem update --system' to update RubyGems itself."
-      end
-    end
-  end
-
   def ensure_dependencies_met # :nodoc:
     deps = spec.runtime_dependencies
     deps |= spec.development_dependencies if @development
@@ -914,8 +893,6 @@ def pre_install_checks
 
     return true if @force
 
-    ensure_required_ruby_version_met
-    ensure_required_rubygems_version_met
     ensure_dependencies_met unless @ignore_dependencies
 
     true

lib/rubygems/remote_fetcher.rb

@@ -215,7 +215,7 @@ def fetch_http(uri, last_modified = nil, head = false, depth = 0)
 
     case response
     when Net::HTTPOK, Net::HTTPNotModified then
-      response.uri = uri if response.respond_to? :uri
+      response.uri = uri
       head ? response : response.body
     when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther,
          Net::HTTPTemporaryRedirect then

lib/rubygems/request_set.rb

@@ -195,19 +195,8 @@ def install(options, &block) # :yields: request, installer
             yield req, installer if block_given?
           end
         rescue Gem::RuntimeRequirementNotMetError => e
-          recent_match = req.spec.set.find_all(req.request).sort_by(&:version).reverse_each.find do |s|
-            s = s.spec
-            s.required_ruby_version.satisfied_by?(Gem.ruby_version) &&
-              s.required_rubygems_version.satisfied_by?(Gem.rubygems_version) &&
-              Gem::Platform.installable?(s)
-          end
-          if recent_match
-            suggestion = "The last version of #{req.request} to support your Ruby & RubyGems was #{recent_match.version}. Try installing it with `gem install #{recent_match.name} -v #{recent_match.version}`"
-            suggestion += " and then running the current command again" unless @always_install.include?(req.spec.spec)
-          else
-            suggestion = "There are no versions of #{req.request} compatible with your Ruby & RubyGems"
-            suggestion += ". Maybe try installing an older version of the gem you're looking for?" unless @always_install.include?(req.spec.spec)
-          end
+          suggestion = "There are no versions of #{req.request} compatible with your Ruby & RubyGems"
+          suggestion += ". Maybe try installing an older version of the gem you're looking for?" unless @always_install.include?(req.spec.spec)
           e.suggestion = suggestion
           raise
         end

lib/rubygems/resolver.rb

@@ -261,7 +261,12 @@ def dependencies_for(specification)
   end
 
   def requirement_satisfied_by?(requirement, activated, spec)
-    requirement.matches_spec? spec
+    matches_spec = requirement.matches_spec? spec
+    return matches_spec if @soft_missing
+
+    matches_spec &&
+      spec.spec.required_ruby_version.satisfied_by?(Gem.ruby_version) &&
+      spec.spec.required_rubygems_version.satisfied_by?(Gem.rubygems_version)
   end
 
   def name_for(dependency)

lib/rubygems/resolver/api_set.rb

@@ -4,6 +4,8 @@
 # Returns instances of APISpecification.
 
 class Gem::Resolver::APISet < Gem::Resolver::Set
+  autoload :GemParser, File.expand_path("api_set/gem_parser", __dir__)
+
   ##
   # The URI for the dependency API this APISet uses.
 
@@ -24,13 +26,13 @@ class Gem::Resolver::APISet < Gem::Resolver::Set
   # API URL +dep_uri+ which is described at
   # https://guides.rubygems.org/rubygems-org-api
 
-  def initialize(dep_uri = 'https://rubygems.org/api/v1/dependencies')
+  def initialize(dep_uri = 'https://index.rubygems.org/info/')
     super()
 
     dep_uri = URI dep_uri unless URI === dep_uri
 
     @dep_uri = dep_uri
-    @uri     = dep_uri + '../..'
+    @uri     = dep_uri + '..'
 
     @data   = Hash.new {|h,k| h[k] = [] }
     @source = Gem::Source.new @uri
@@ -75,20 +77,8 @@ def prefetch(reqs)
   def prefetch_now # :nodoc:
     needed, @to_fetch = @to_fetch, []
 
-    uri = @dep_uri + "?gems=#{needed.sort.join ','}"
-    str = Gem::RemoteFetcher.fetcher.fetch_path uri
-
-    loaded = []
-
-    Marshal.load(str).each do |ver|
-      name = ver[:name]
-
-      @data[name] << ver
-      loaded << name
-    end
-
-    (needed - loaded).each do |missing|
-      @data[missing] = []
+    needed.sort.each do |name|
+      versions(name)
     end
   end
 
@@ -111,13 +101,32 @@ def versions(name) # :nodoc:
       return @data[name]
     end
 
-    uri = @dep_uri + "?gems=#{name}"
+    uri = @dep_uri + name
     str = Gem::RemoteFetcher.fetcher.fetch_path uri
 
-    Marshal.load(str).each do |ver|
-      @data[ver[:name]] << ver
+    lines(str).each do |ver|
+      number, platform, dependencies, requirements = parse_gem(ver)
+
+      platform ||= "ruby"
+      dependencies = dependencies.map {|dep_name, reqs| [dep_name, reqs.join(", ")] }
+      requirements = requirements.map {|req_name, reqs| [req_name.to_sym, reqs] }.to_h
+
+      @data[name] << { name: name, number: number, platform: platform, dependencies: dependencies, requirements: requirements }
     end
 
     @data[name]
   end
+
+  private
+
+  def lines(str)
+    lines = str.split("\n")
+    header = lines.index("---")
+    header ? lines[header + 1..-1] : lines
+  end
+
+  def parse_gem(string)
+    @gem_parser ||= GemParser.new
+    @gem_parser.parse(string)
+  end
 end

lib/rubygems/resolver/api_set/gem_parser.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class Gem::Resolver::APISet::GemParser
+  def parse(line)
+    version_and_platform, rest = line.split(" ", 2)
+    version, platform = version_and_platform.split("-", 2)
+    dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest
+    dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : []
+    requirements = requirements ? requirements.map {|d| parse_dependency(d) } : []
+    [version, platform, dependencies, requirements]
+  end
+
+  private
+
+  def parse_dependency(string)
+    dependency = string.split(":")
+    dependency[-1] = dependency[-1].split("&") if dependency.size > 1
+    dependency
+  end
+end

lib/rubygems/resolver/api_specification.rb

@@ -35,19 +35,20 @@ def initialize(set, api_data)
     @dependencies = api_data[:dependencies].map do |name, ver|
       Gem::Dependency.new(name, ver.split(/\s*,\s*/)).freeze
     end.freeze
+    @required_ruby_version = Gem::Requirement.new(api_data.dig(:requirements, :ruby)).freeze
+    @required_rubygems_version = Gem::Requirement.new(api_data.dig(:requirements, :rubygems)).freeze
   end
 
   def ==(other) # :nodoc:
     self.class === other and
       @set          == other.set and
       @name         == other.name and
       @version      == other.version and
-      @platform     == other.platform and
-      @dependencies == other.dependencies
+      @platform     == other.platform
   end
 
   def hash
-    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash ^ @dependencies.hash
+    @set.hash ^ @name.hash ^ @version.hash ^ @platform.hash
   end
 
   def fetch_development_dependencies # :nodoc:

lib/rubygems/resolver/best_set.rb

@@ -60,7 +60,7 @@ def pretty_print(q) # :nodoc:
   def replace_failed_api_set(error) # :nodoc:
     uri = error.uri
     uri = URI uri unless URI === uri
-    uri.query = nil
+    uri = uri + "."
 
     raise error unless api_set = @sets.find do |set|
       Gem::Resolver::APISet === set and set.dep_uri == uri

lib/rubygems/resolver/index_specification.rb

@@ -33,6 +33,21 @@ def dependencies
     spec.dependencies
   end
 
+  ##
+  # The required_ruby_version constraint for this specification
+
+  def required_ruby_version
+    spec.required_ruby_version
+  end
+
+  ##
+  # The required_rubygems_version constraint for this specification
+  #
+
+  def required_rubygems_version
+    spec.required_rubygems_version
+  end
+
   def ==(other)
     self.class === other &&
       @name == other.name &&