internal/database: (no-op) split load and generate files

Move Diff function into its own file, and move code and tests related
to OSV generation into their own file.

For golang/go#56417

Change-Id: Ia41b3f9068efe543c1a603c34738810c404e6caf
Reviewed-on: https://go-review.googlesource.com/c/vulndb/+/448840
Run-TryBot: Tatiana Bradley <[email protected]>
Reviewed-by: Damien Neil <[email protected]>
Reviewed-by: Jonathan Amsterdam <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>

internal/database/diff.go

@@ -0,0 +1,34 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package database
+
+import (
+	"fmt"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/vulndb/internal/derrors"
+)
+
+func Diff(dbname1, dbname2 string) (err error) {
+	defer derrors.Wrap(&err, "Diff(%q, %q)", dbname1, dbname2)
+	indexA, dbA, err := Load(dbname1)
+	if err != nil {
+		return fmt.Errorf("unable to load %q: %s", dbname1, err)
+	}
+	indexB, dbB, err := Load(dbname2)
+	if err != nil {
+		return fmt.Errorf("unable to load %q: %s", dbname2, err)
+	}
+	indexDiff := cmp.Diff(indexA, indexB)
+	if indexDiff == "" {
+		indexDiff = "(no change)"
+	}
+	dbDiff := cmp.Diff(dbA, dbB)
+	if dbDiff == "" {
+		dbDiff = "(no change)"
+	}
+	fmt.Printf("# index\n%s\n\n# db\n%s\n", indexDiff, dbDiff)
+	return nil
+}

internal/database/generate.go

@@ -11,18 +11,12 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"regexp"
-	"sort"
 	"strings"
-	"time"
 
-	"golang.org/x/exp/maps"
 	"golang.org/x/vuln/client"
 	"golang.org/x/vuln/osv"
 	"golang.org/x/vulndb/internal/derrors"
 	"golang.org/x/vulndb/internal/gitrepo"
-	"golang.org/x/vulndb/internal/report"
-	"golang.org/x/vulndb/internal/stdlib"
 )
 
 const (
@@ -129,15 +123,6 @@ func generateEntries(ctx context.Context, repoDir string) (map[string][]osv.Entr
 	return jsonVulns, entries, nil
 }
 
-// ModulesForEntry returns the list of modules affected by an OSV entry.
-func ModulesForEntry(entry osv.Entry) []string {
-	mods := map[string]bool{}
-	for _, a := range entry.Affected {
-		mods[a.Package.Name] = true
-	}
-	return maps.Keys(mods)
-}
-
 func writeVulns(outPath string, vulns []osv.Entry, indent bool) error {
 	if err := os.MkdirAll(filepath.Dir(outPath), 0755); err != nil {
 		return fmt.Errorf("failed to create directory %q: %s", filepath.Dir(outPath), err)
@@ -189,119 +174,3 @@ func jsonMarshal(v any, indent bool) ([]byte, error) {
 	}
 	return json.Marshal(v)
 }
-
-// ReadOSV reads an osv.Entry from a file.
-func ReadOSV(filename string) (osv.Entry, error) {
-	b, err := os.ReadFile(filename)
-	if err != nil {
-		return osv.Entry{}, err
-	}
-	var entry osv.Entry
-	if err := json.Unmarshal(b, &entry); err != nil {
-		return osv.Entry{}, fmt.Errorf("%v: %w", filename, err)
-	}
-	return entry, nil
-}
-
-// GenerateOSVEntry create an osv.Entry for a report. In addition to the report, it
-// takes the ID for the vuln and a URL that will point to the entry in the vuln DB.
-func GenerateOSVEntry(filename string, lastModified time.Time, r *report.Report) osv.Entry {
-	id := report.GetGoIDFromFilename(filename)
-
-	var credits []osv.Credit
-	if r.Credit != "" {
-		credits = append(credits, osv.Credit{
-			Name: r.Credit,
-		})
-	}
-
-	entry := osv.Entry{
-		ID:        id,
-		Published: r.Published,
-		Modified:  lastModified,
-		Withdrawn: r.Withdrawn,
-		Details:   trimWhitespace(r.Description),
-		Credits:   credits,
-	}
-
-	linkName := report.GetGoAdvisoryLink(id)
-	for _, m := range r.Modules {
-		entry.Affected = append(entry.Affected, generateAffected(m, linkName))
-	}
-	for _, ref := range r.References {
-		entry.References = append(entry.References, osv.Reference{
-			Type: string(ref.Type),
-			URL:  ref.URL,
-		})
-	}
-	entry.Aliases = r.GetAliases()
-	return entry
-}
-
-func generateAffectedRanges(versions []report.VersionRange) osv.Affects {
-	a := osv.AffectsRange{Type: osv.TypeSemver}
-	if len(versions) == 0 || versions[0].Introduced == "" {
-		a.Events = append(a.Events, osv.RangeEvent{Introduced: "0"})
-	}
-	for _, v := range versions {
-		if v.Introduced != "" {
-			a.Events = append(a.Events, osv.RangeEvent{Introduced: v.Introduced.Canonical()})
-		}
-		if v.Fixed != "" {
-			a.Events = append(a.Events, osv.RangeEvent{Fixed: v.Fixed.Canonical()})
-		}
-	}
-	return osv.Affects{a}
-}
-
-// trimWhitespace removes unnecessary whitespace from a string, but preserves
-// paragraph breaks (indicated by two newlines).
-func trimWhitespace(s string) string {
-	s = strings.TrimSpace(s)
-	// Replace single newlines with spaces.
-	newlines := regexp.MustCompile(`([^\n])\n([^\n])`)
-	s = newlines.ReplaceAllString(s, "$1 $2")
-	// Replace instances of 2 or more newlines with exactly two newlines.
-	paragraphs := regexp.MustCompile(`\s*\n\n\s*`)
-	s = paragraphs.ReplaceAllString(s, "\n\n")
-	// Replace tabs and double spaces with single spaces.
-	spaces := regexp.MustCompile(`[ \t]+`)
-	s = spaces.ReplaceAllString(s, " ")
-	return s
-}
-
-func generateImports(m *report.Module) (imps []osv.EcosystemSpecificImport) {
-	for _, p := range m.Packages {
-		syms := append([]string{}, p.Symbols...)
-		syms = append(syms, p.DerivedSymbols...)
-		sort.Strings(syms)
-		imps = append(imps, osv.EcosystemSpecificImport{
-			Path:    p.Package,
-			GOOS:    p.GOOS,
-			GOARCH:  p.GOARCH,
-			Symbols: syms,
-		})
-	}
-	return imps
-}
-
-func generateAffected(m *report.Module, url string) osv.Affected {
-	name := m.Module
-	switch name {
-	case stdlib.ModulePath:
-		name = stdFileName
-	case stdlib.ToolchainModulePath:
-		name = toolchainFileName
-	}
-	return osv.Affected{
-		Package: osv.Package{
-			Name:      name,
-			Ecosystem: osv.GoEcosystem,
-		},
-		Ranges:           generateAffectedRanges(m.Versions),
-		DatabaseSpecific: osv.DatabaseSpecific{URL: url},
-		EcosystemSpecific: osv.EcosystemSpecific{
-			Imports: generateImports(m),
-		},
-	}
-}