Merge pull request #2880 from dependabot/jurre-fatih/stop-parsing-unr…

…equired-indirect-go-deps

gomod: Do not attempt to parse transitive dependencies

go_modules/lib/dependabot/go_modules/file_parser.rb

@@ -16,17 +16,8 @@ class FileParser < Dependabot::FileParsers::Base
       def parse
         dependency_set = Dependabot::FileParsers::Base::DependencySet.new
 
-        i = 0
-        chunks = module_info.lines.
-                 group_by { |line| line == "{\n" ? i += 1 : i }
-        deps = chunks.values.map { |chunk| JSON.parse(chunk.join) }
-
-        deps.each do |dep|
-          # The project itself appears in this list as "Main"
-          next if dep["Main"]
-
-          dependency = dependency_from_details(dep)
-          dependency_set << dependency if dependency
+        required_packages.each do |dep|
+          dependency_set << dependency_from_details(dep) unless dep["Indirect"]
         end
 
         dependency_set.dependencies
@@ -65,39 +56,36 @@ def dependency_from_details(details)
         )
       end
 
-      def module_info
-        @module_info ||=
+      def required_packages
+        @required_packages ||=
           SharedHelpers.in_a_temporary_directory do |path|
-            SharedHelpers.with_git_configured(credentials: credentials) do
-              # Create a fake empty module for each local module so that
-              # `go list` works, even if some modules have been `replace`d with
-              # a local module that we don't have access to.
-              local_replacements.each do |_, stub_path|
-                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                FileUtils.touch(File.join(stub_path, "go.mod"))
-              end
-
-              File.write("go.mod", go_mod_content)
-
-              command = "go mod edit -print > /dev/null"
-              command += " && go list -m -json all"
-
-              # Turn off the module proxy for now, as it's causing issues with
-              # private git dependencies
-              env = { "GOPRIVATE" => "*" }
-
-              stdout, stderr, status = Open3.capture3(env, command)
-              handle_parser_error(path, stderr) unless status.success?
-              stdout
-            rescue Dependabot::DependencyFileNotResolvable
-              # We sometimes see this error if a host times out.
-              # In such cases, retrying (a maximum of 3 times) may fix it.
-              retry_count ||= 0
-              raise if retry_count >= 3
-
-              retry_count += 1
-              retry
+            # Create a fake empty module for each local module so that
+            # `go mod edit` works, even if some modules have been `replace`d with
+            # a local module that we don't have access to.
+            local_replacements.each do |_, stub_path|
+              Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+              FileUtils.touch(File.join(stub_path, "go.mod"))
             end
+
+            File.write("go.mod", go_mod_content)
+
+            command = "go mod edit -json"
+
+            # Turn off the module proxy for now, as it's causing issues with
+            # private git dependencies
+            env = { "GOPRIVATE" => "*" }
+
+            stdout, stderr, status = Open3.capture3(env, command)
+            handle_parser_error(path, stderr) unless status.success?
+            JSON.parse(stdout)["Require"]
+          rescue Dependabot::DependencyFileNotResolvable
+            # We sometimes see this error if a host times out.
+            # In such cases, retrying (a maximum of 3 times) may fix it.
+            retry_count ||= 0
+            raise if retry_count >= 3
+
+            retry_count += 1
+            retry
           end
       end
 
@@ -135,52 +123,9 @@ def go_mod_content
         end
       end
 
-      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/m.freeze
       def handle_parser_error(path, stderr)
-        case stderr
-        when /go: .*: unknown revision/m
-          line = stderr.lines.grep(/unknown revision/).first.strip
-          handle_github_unknown_revision(line) if line.start_with?("go: github.com/")
-          raise Dependabot::DependencyFileNotResolvable, line
-        when /go: .*: unrecognized import path/m
-          line = stderr.lines.grep(/unrecognized import/).first
-          raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: errors parsing go.mod/m
-          msg = stderr.gsub(path.to_s, "").strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
-        when GIT_ERROR_REGEX
-          lines = stderr.lines.drop_while { |l| GIT_ERROR_REGEX !~ l }
-          raise Dependabot::DependencyFileNotResolvable.new, lines.join
-        else
-          msg = stderr.gsub(path.to_s, "").strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
-        end
-      end
-
-      GITHUB_REPO_REGEX = %r{github.com/[^@]*}.freeze
-      def handle_github_unknown_revision(line)
-        mod_path = line.scan(GITHUB_REPO_REGEX).first
-        return unless mod_path
-
-        # Query for _any_ version of this module, to know if it doesn't exist (or is private)
-        # or we were just given a bad revision by this manifest
-        SharedHelpers.in_a_temporary_directory do
-          SharedHelpers.with_git_configured(credentials: credentials) do
-            File.write("go.mod", "module dummy\n")
-
-            env = { "GOPRIVATE" => "*" }
-            _, _, status = Open3.capture3(env, SharedHelpers.escape_command("go get #{mod_path}"))
-            raise Dependabot::DependencyFileNotResolvable, line if status.success?
-
-            mod_split = mod_path.split("/")
-            repo_path = if mod_split.size > 3
-                          mod_split[0..2].join("/")
-                        else
-                          mod_path
-                        end
-            raise Dependabot::GitDependenciesNotReachable, [repo_path]
-          end
-        end
+        msg = stderr.gsub(path.to_s, "").strip
+        raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
       end
 
       def rev_identifier?(dep)

go_modules/spec/dependabot/go_modules/file_parser_spec.rb

@@ -37,7 +37,7 @@
   describe "parse" do
     subject(:dependencies) { parser.parse }
 
-    its(:length) { is_expected.to eq(6) }
+    its(:length) { is_expected.to eq(3) }
 
     describe "top level dependencies" do
       subject(:dependencies) do
@@ -144,9 +144,8 @@
         go_mod.sub("rsc.io/quote", "example.com/not-a-repo")
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::DependencyFileNotResolvable)
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -157,11 +156,8 @@
         go_mod.sub("rsc.io/quote", invalid_repo)
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::GitDependenciesNotReachable) do |error|
-            expect(error.dependency_urls).to contain_exactly(invalid_repo)
-          end
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -172,11 +168,8 @@
         go_mod.sub("rsc.io/quote v1.4.0", "#{invalid_repo}/v2 v2.0.0")
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::GitDependenciesNotReachable) do |error|
-            expect(error.dependency_urls).to contain_exactly(invalid_repo)
-          end
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -186,9 +179,8 @@
         go_mod.sub("github.com/mattn/go-colorable v0.0.9", "github.com/mattn/go-colorable v0.1234.4321")
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::DependencyFileNotResolvable)
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -197,10 +189,7 @@
       let(:go_mod_fixture_name) { "parent_module.mod" }
 
       it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
-            expect(error.message).to include("hmarr/404")
-          end
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -210,9 +199,8 @@
         go_mod.sub("rsc.io/quote v1.4.0", "rsc.io/quote v1.321.0")
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::DependencyFileNotResolvable)
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end
 
@@ -223,11 +211,8 @@
                    "github.com/hmarr/404 v0.0.0-20181216014959-b89dc648a159")
       end
 
-      it "raises the correct error" do
-        expect { parser.parse }.
-          to raise_error(Dependabot::DependencyFileNotResolvable) do |error|
-            expect(error.message).to include("hmarr/404")
-          end
+      it "does not raise an error" do
+        expect { parser.parse }.not_to raise_error
       end
     end