Add osv output lockfile + refactor (#505)

Introduces the osv-scanner results lockfile format (feel free to suggest
a better name for this),

This also refactors the models folder slightly to move some of the
helper functions out to a separate internal crate. This stops the models
package from pulling in the lockfile package, causing cyclic imports.

I also added another field to `PackageDetails`, `Source`, a currently
optional string to specify the location the dependency is actually
specified. The only times it will be different from the lockfile
location currently is:

- Python requirements.txt `-r`
- This new osv-scanner format, it stores the lockfile path parsed in the
output results.

pkg/lockfile/extractor.go

@@ -60,6 +60,9 @@ func OpenLocalDepFile(path string) (NestedDepFile, error) {
 		return LocalFile{}, err
 	}
 
+	// Very unlikely to have Abs return an error if the file opens correctly
+	path, _ = filepath.Abs(path)
+
 	return LocalFile{r, path}, nil
 }
 

pkg/lockfile/fixtures/osvscannerresults/empty.json

@@ -0,0 +1,3 @@
+{
+    "results": []
+}

pkg/lockfile/fixtures/osvscannerresults/not-json.txt

@@ -0,0 +1 @@
+this is not valid json! (I think)

pkg/lockfile/fixtures/osvscannerresults/one-package.json

@@ -0,0 +1,21 @@
+{
+  "results": [
+    {
+      "source": {
+        "path": "/path/to/Gemfile.lock",
+        "type": "lockfile"
+      },
+      "packages": [
+        {
+          "package": {
+            "name": "activesupport",
+            "version": "7.0.7",
+            "ecosystem": "RubyGems"
+          },
+          "vulnerabilities": [],
+          "groups": []
+        }
+      ]
+    }
+  ]
+}

pkg/lockfile/helpers_test.go

@@ -31,7 +31,9 @@ func packageToString(pkg lockfile.PackageDetails) string {
 	return fmt.Sprintf("%s@%s (%s, %s)", pkg.Name, pkg.Version, pkg.Ecosystem, commit)
 }
 
-func hasPackage(packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool {
+func hasPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) bool {
+	t.Helper()
+
 	for _, details := range packages {
 		if details == pkg {
 			return true
@@ -44,7 +46,7 @@ func hasPackage(packages []lockfile.PackageDetails, pkg lockfile.PackageDetails)
 func expectPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfile.PackageDetails) {
 	t.Helper()
 
-	if !hasPackage(packages, pkg) {
+	if !hasPackage(t, packages, pkg) {
 		t.Errorf(
 			"Expected packages to include %s@%s (%s, %s), but it did not",
 			pkg.Name,
@@ -55,11 +57,12 @@ func expectPackage(t *testing.T, packages []lockfile.PackageDetails, pkg lockfil
 	}
 }
 
-func findMissingPackages(actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) []lockfile.PackageDetails {
+func findMissingPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expectedPackages []lockfile.PackageDetails) []lockfile.PackageDetails {
+	t.Helper()
 	var missingPackages []lockfile.PackageDetails
 
 	for _, pkg := range actualPackages {
-		if !hasPackage(expectedPackages, pkg) {
+		if !hasPackage(t, expectedPackages, pkg) {
 			missingPackages = append(missingPackages, pkg)
 		}
 	}
@@ -79,8 +82,8 @@ func expectPackages(t *testing.T, actualPackages []lockfile.PackageDetails, expe
 		)
 	}
 
-	missingActualPackages := findMissingPackages(actualPackages, expectedPackages)
-	missingExpectedPackages := findMissingPackages(expectedPackages, actualPackages)
+	missingActualPackages := findMissingPackages(t, actualPackages, expectedPackages)
+	missingExpectedPackages := findMissingPackages(t, expectedPackages, actualPackages)
 
 	if len(missingActualPackages) != 0 {
 		for _, unexpectedPackage := range missingActualPackages {

pkg/lockfile/osv-vuln-result_test.go

@@ -0,0 +1,87 @@
+package lockfile_test
+
+import (
+	"testing"
+
+	"github.com/google/osv-scanner/pkg/lockfile"
+)
+
+func TestParseOSVScannerResults_FileDoesNotExist(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/does-not-exist")
+
+	expectErrContaining(t, err, "no such file or directory")
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_InvalidJSON(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/not-json.txt")
+
+	expectErrContaining(t, err, "could not extract from")
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_NoPackages(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/empty.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{})
+}
+
+func TestParseOSVScannerResults_OnePackage(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/one-package.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "activesupport",
+			Version:   "7.0.7",
+			Ecosystem: lockfile.BundlerEcosystem,
+			CompareAs: lockfile.BundlerEcosystem,
+		},
+	})
+}
+
+func TestParseOSVScannerResults_MultiPackages(t *testing.T) {
+	t.Parallel()
+
+	packages, err := lockfile.ParseOSVScannerResults("fixtures/osvscannerresults/multi-packages-with-vulns.json")
+
+	if err != nil {
+		t.Errorf("Got unexpected error: %v", err)
+	}
+
+	expectPackages(t, packages, []lockfile.PackageDetails{
+		{
+			Name:      "crossbeam-utils",
+			Version:   "0.6.6",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+		},
+		{
+			Name:      "memoffset",
+			Version:   "0.5.6",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+		},
+		{
+			Name:      "smallvec",
+			Version:   "1.6.0",
+			Ecosystem: lockfile.CargoEcosystem,
+			CompareAs: lockfile.CargoEcosystem,
+		},
+	})
+}

pkg/lockfile/osv-vuln-results.go

@@ -0,0 +1,55 @@
+package lockfile
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/google/osv-scanner/pkg/models"
+)
+
+func ParseOSVScannerResults(pathToLockfile string) ([]PackageDetails, error) {
+	return extractFromFile(pathToLockfile, OSVScannerResultsExtractor{})
+}
+
+type OSVScannerResultsExtractor struct{}
+
+func (e OSVScannerResultsExtractor) ShouldExtract(path string) bool {
+	// The output will always be a custom json file, so don't return a default should extract
+	return false
+}
+
+func (e OSVScannerResultsExtractor) Extract(f DepFile) ([]PackageDetails, error) {
+	parsedResults := models.VulnerabilityResults{}
+	err := json.NewDecoder(f).Decode(&parsedResults)
+
+	if err != nil {
+		return nil, fmt.Errorf("could not extract from %s: %w", f.Path(), err)
+	}
+
+	packages := []PackageDetails{}
+	for _, res := range parsedResults.Results {
+		for _, pkg := range res.Packages {
+			packages = append(packages, PackageDetails{
+				Name:      pkg.Package.Name,
+				Ecosystem: Ecosystem(pkg.Package.Ecosystem),
+				Version:   pkg.Package.Version,
+				CompareAs: Ecosystem(pkg.Package.Ecosystem),
+			})
+		}
+	}
+
+	return packages, nil
+}
+
+var _ Extractor = OSVScannerResultsExtractor{}
+
+// FromOSVScannerResults attempts to extract packages stored in the OSVScannerResults format
+func FromOSVScannerResults(pathToInstalled string) (Lockfile, error) {
+	packages, err := extractFromFile(pathToInstalled, OSVScannerResultsExtractor{})
+
+	return Lockfile{
+		FilePath: pathToInstalled,
+		ParsedAs: "osv-scanner-results",
+		Packages: packages,
+	}, err
+}

pkg/lockfile/parse-requirements-txt.go

@@ -167,6 +167,7 @@ func parseRequirementsTxt(f DepFile, requiredAlready map[string]struct{}) ([]Pac
 		}
 
 		detail := parseLine(line)
+
 		packages[detail.Name+"@"+detail.Version] = detail
 	}