Add osv output lockfile + refactor

cmd/osv-scanner/main_test.go

@@ -91,6 +91,8 @@ type cliTestCase struct {
 func testCli(t *testing.T, tc cliTestCase) {
 	t.Helper()
 
+	t.Logf("Running test: [%s]", tc.name)
+
 	stdoutBuffer := &bytes.Buffer{}
 	stderrBuffer := &bytes.Buffer{}
 

pkg/lockfile/extractor.go

@@ -60,6 +60,9 @@ func OpenLocalDepFile(path string) (NestedDepFile, error) {
 		return LocalFile{}, err
 	}
 
+	// Very unlikely to have Abs return an error if the file opens correctly
+	path, _ = filepath.Abs(path)
+
 	return LocalFile{r, path}, nil
 }
 

pkg/lockfile/fixtures/osvscannerresults/empty.json

@@ -0,0 +1,3 @@
+{
+    "results": []
+}

pkg/lockfile/fixtures/osvscannerresults/not-json.txt

@@ -0,0 +1 @@
+this is not valid json! (I think)

pkg/lockfile/fixtures/osvscannerresults/one-package.json

@@ -0,0 +1,21 @@
+{
+  "results": [
+    {
+      "source": {
+        "path": "/path/to/Gemfile.lock",
+        "type": "lockfile"
+      },
+      "packages": [
+        {
+          "package": {
+            "name": "activesupport",
+            "version": "7.0.7",
+            "ecosystem": "RubyGems"
+          },
+          "vulnerabilities": [],
+          "groups": []
+        }
+      ]
+    }
+  ]
+}

pkg/lockfile/helpers_test.go

@@ -2,6 +2,8 @@ package lockfile_test
 
 import (
 	"fmt"
+	"log"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -31,7 +33,9 @@ func packageToString(pkg lockfile.PackageDetails) string {
 	return fmt.Sprintf("%s@%s (%s, %s)", pkg.Name, pkg.Version, pkg.Ecosystem, commit)
 }
 
-func hasPackage(packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool {
+func hasPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool {
+	t.Helper()
+
 	for _, details := range packages {
 		if details == pkg {
 			return true
@@ -41,10 +45,20 @@ func hasPackage(packages []lockfile.PackageDetails, pkg lockfile.PackageDetails)
 	return false
 }
 
+func absPathNoErr(path string) string {
+	result, err := filepath.Abs(path)
+
+	if err != nil {
+		log.Panicf("failed to find abs path: %s", err)
+	}
+
+	return result
+}
+
 func expectPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) {
 	t.Helper()
 
-	if !hasPackage(packages, pkg) {
+	if !hasPackage(t, packages, pkg) {
 		t.Errorf(
 			"Expected packages to include %s@%s (%s, %s), but it did not",
 			pkg.Name,
@@ -55,11 +69,12 @@ func expectPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfil
 	}
 }
 
-func findMissingPackages(actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) []lockfile.PackageDetails {
+func findMissingPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) []lockfile.PackageDetails {
+	t.Helper()
 	var missingPackages []lockfile.PackageDetails
 
 	for _, pkg := range actualPackages {
-		if !hasPackage(expectedPackages, pkg) {
+		if !hasPackage(t, expectedPackages, pkg) {
 			missingPackages = append(missingPackages, pkg)
 		}
 	}
@@ -79,8 +94,8 @@ func expectPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expe
 		)
 	}
 
-	missingActualPackages := findMissingPackages(actualPackages, expectedPackages)
-	missingExpectedPackages := findMissingPackages(expectedPackages, actualPackages)
+	missingActualPackages := findMissingPackages(t, actualPackages, expectedPackages)
+	missingExpectedPackages := findMissingPackages(t, expectedPackages, actualPackages)
 
 	if len(missingActualPackages) != 0 {
 		for _, unexpectedPackage := range missingActualPackages {

pkg/lockfile/osv-vuln-result_test.go

@@ -0,0 +1,91 @@
+package lockfile_test
+
+import (
+	"testing"
+
+	"github.com/google/osv-scanner/pkg/lockfile"
+)
+
+func TestParseOSVScannerResults_FileDoesNotExist(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/does-not-exist")
+
+	expectErrContaining(t, err, "no such file or directory")
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_InvalidJSON(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/not-json.txt")
+
+	expectErrContaining(t, err, "could not extract from")
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_NoPackages(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/empty.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_OnePackage(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/one-package.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "activesupport",
+			Version:   "7.0.7",
+			Ecosystem: lockfile.BundlerEcosystem,
+			CompareAs: lockfile.BundlerEcosystem,
+			Source:    "/path/to/Gemfile.lock",
+		},
+	})
+}
+
+func TestParseOSVScannerResults_MultiPackages(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/multi-packages-with-vulns.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "crossbeam-utils",
+			Version:   "0.6.6",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+			Source:    "/path/to/Cargo.lock",
+		},
+		{
+			Name:      "memoffset",
+			Version:   "0.5.6",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+			Source:    "/path/to/Cargo.lock",
+		},
+		{
+			Name:      "smallvec",
+			Version:   "1.6.0",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+			Source:    "/path/to/Cargo.lock",
+		},
+	})
+}

pkg/lockfile/osv-vuln-results.go

@@ -0,0 +1,56 @@
+package lockfile
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/osv-scanner/pkg/models"
+)
+
+func ParseOSVScannerResults(pathToLockfile string) ([]PackageDetails, error) {
+	return extractFromFile(pathToLockfile, OSVScannerResultsExtractor{})
+}
+
+type OSVScannerResultsExtractor struct{}
+
+func (e OSVScannerResultsExtractor) ShouldExtract(path string) bool {
+	// The output will always be a custom json file, so don't return a default should extract
+	return false
+}
+
+func (e OSVScannerResultsExtractor) Extract(f DepFile) ([]PackageDetails, error) {
+	parsedResults := models.VulnerabilityResults{}
+	err := json.NewDecoder(f).Decode(&parsedResults)
+
+	if err != nil {
+		return nil, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+	}
+
+	packages := []PackageDetails{}
+	for _, res := range parsedResults.Results {
+		for _, pkg := range res.Packages {
+			packages = append(packages, PackageDetails{
+				Name:      pkg.Package.Name,
+				Ecosystem: Ecosystem(pkg.Package.Ecosystem),
+				Version:   pkg.Package.Version,
+				CompareAs: Ecosystem(pkg.Package.Ecosystem),
+				Source:    res.Source.Path,
+			})
+		}
+	}
+
+	return packages, nil
+}
+
+var _ Extractor = OSVScannerResultsExtractor{}
+
+// FromOSVScannerResults attempts to extract packages stored in the OSVScannerResults format
+func FromOSVScannerResults(pathToInstalled string) (Lockfile, error) {
+	packages, err := extractFromFile(pathToInstalled, OSVScannerResultsExtractor{})
+
+	return Lockfile{
+		FilePath: pathToInstalled,
+		ParsedAs: "osv-scanner-results",
+		Packages: packages,
+	}, err
+}

pkg/lockfile/parse-requirements-txt.go

@@ -167,6 +167,9 @@ func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]Pac
 		}
 
 		detail := parseLine(line)
+		// Fill out source for requirements.txt because dependencies might not be in the initial lockfile
+		// that's passed
+		detail.Source = f.Path()
 		packages[detail.Name+"@"+detail.Version] = detail
 	}