From 5c160367e2062c5cf6d467fe2e48dbec9d829212 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Wed, 4 Oct 2023 10:07:08 -0400 Subject: [PATCH 01/11] feat: add cpe generation for golang compiler Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/catalog.go | 1 - syft/pkg/cataloger/golang/package.go | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index 840ea72b7aa..841a4c28baf 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -92,7 +92,6 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes p.Language = pkg.LanguageFromPURL(p.PURL) } - // create file-to-package relationships for files owned by the package owningRelationships, err := packageFileOwnershipRelationships(p, resolver) if err != nil { log.WithFields("cataloger", cataloger.Name(), "package", p.Name, "error", err).Warnf("unable to create any package-file relationships") diff --git a/syft/pkg/cataloger/golang/package.go b/syft/pkg/cataloger/golang/package.go index 37942563bf3..a66154f2ceb 100644 --- a/syft/pkg/cataloger/golang/package.go +++ b/syft/pkg/cataloger/golang/package.go @@ -1,12 +1,14 @@ package golang import ( + "fmt" "regexp" "runtime/debug" "strings" "github.com/anchore/packageurl-go" "github.com/anchore/syft/internal/log" + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -21,11 +23,20 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err) } + cpes := make([]cpe.CPE, 0) + compilerCpe, err := buildCompilerCPE(goVersion) + if err != nil { + log.Warnf("unable to build CPE for golang compiler for package %s", dep.Path) + } else { + cpes = append(cpes, compilerCpe) + } + p := pkg.Package{ Name: dep.Path, Version: dep.Version, Licenses: pkg.NewLicenseSet(licenses...), PURL: packageURL(dep.Path, dep.Version), + CPEs: cpes, Language: pkg.Go, Type: pkg.GoModulePkg, Locations: file.NewLocationSet(locations...), @@ -45,6 +56,14 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu return p } +func buildCompilerCPE(goVersion string) (cpe.CPE, error) { + cpe, err := cpe.New(fmt.Sprintf("pkg:golang/stdlib@%s", goVersion)) + if err != nil { + + } + return cpe, nil +} + func packageURL(moduleName, moduleVersion string) string { // source: https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#golang // note: "The version is often empty when a commit is not specified and should be the commit in most cases when available." From 315c57a573e4c329f7d0341987dc44502a275d64 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Wed, 4 Oct 2023 10:57:58 -0400 Subject: [PATCH 02/11] feat: append CPE for detecting go compiler version Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/package.go | 9 ---- syft/pkg/cataloger/golang/parse_go_binary.go | 46 +++++++++++++++++--- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/syft/pkg/cataloger/golang/package.go b/syft/pkg/cataloger/golang/package.go index a66154f2ceb..093d5de16fd 100644 --- a/syft/pkg/cataloger/golang/package.go +++ b/syft/pkg/cataloger/golang/package.go @@ -23,20 +23,11 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu log.Tracef("error getting licenses for golang package: %s %v", dep.Path, err) } - cpes := make([]cpe.CPE, 0) - compilerCpe, err := buildCompilerCPE(goVersion) - if err != nil { - log.Warnf("unable to build CPE for golang compiler for package %s", dep.Path) - } else { - cpes = append(cpes, compilerCpe) - } - p := pkg.Package{ Name: dep.Path, Version: dep.Version, Licenses: pkg.NewLicenseSet(licenses...), PURL: packageURL(dep.Path, dep.Version), - CPEs: cpes, Language: pkg.Go, Type: pkg.GoModulePkg, Locations: file.NewLocationSet(locations...), diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index 5286270e1ce..c650121e73a 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -17,6 +17,7 @@ import ( "github.com/anchore/syft/internal" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" @@ -59,12 +60,45 @@ func (c *goBinaryCataloger) parseGoBinary(resolver file.Resolver, _ *generic.Env mods := scanFile(unionReader, reader.RealPath) internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) + compilerVersions := make(map[string]interface{}) + for _, mod := range mods { - pkgs = append(pkgs, c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch)...) + goPkgs, goCompilerVersion := c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch) + compilerVersions[goCompilerVersion] = struct{}{} + pkgs = append(pkgs, goPkgs...) + } + + for key, _ := range compilerVersions { + pkg := newGoStdLib(key) + pkg.SetID() + pkgs = append(pkgs, pkg) } return pkgs, nil, nil } +func newGoStdLib(version string) pkg.Package { + version = strings.TrimPrefix(version, "go") + cpes := make([]cpe.CPE, 0) + compilerCPE, err := cpe.New(fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", version)) + if err != nil { + /// + } else { + cpes = append(cpes, compilerCPE) + } + pkg := pkg.Package{ + Name: "Golang Standard Library", + Version: version, + PURL: packageURL("stdlib", version), + CPEs: cpes, + Language: pkg.Go, + Type: pkg.GoModulePkg, + MetadataType: pkg.GolangBinMetadataType, + Metadata: pkg.GolangBinMetadata{}, + } + + return pkg +} + func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location) pkg.Package { gbs := getBuildSettings(mod.Settings) main := c.newGoBinaryPackage( @@ -219,10 +253,10 @@ func createMainModuleFromPath(path string) (mod debug.Module) { return } -func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string) []pkg.Package { - var pkgs []pkg.Package +func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string) (pkgs []pkg.Package, goCompilerVersion string) { + pkgs = make([]pkg.Package, 0) if mod == nil { - return pkgs + return pkgs, "" } var empty debug.Module @@ -250,11 +284,11 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file } if mod.Main == empty { - return pkgs + return pkgs, "" } main := c.makeGoMainPackage(resolver, mod, arch, location) pkgs = append(pkgs, main) - return pkgs + return pkgs, mod.GoVersion } From a1894b5a58692d79cd55992d94221a631ba682d3 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Wed, 4 Oct 2023 12:04:24 -0400 Subject: [PATCH 03/11] test: add integration test for detectecing go compiler version Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/parse_go_binary.go | 2 +- .../integration/go_compiler_detection_test.go | 36 +++++++++++++++++++ .../image-golang-compiler/Dockerfile | 1 + 3 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 test/integration/go_compiler_detection_test.go create mode 100644 test/integration/test-fixtures/image-golang-compiler/Dockerfile diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index c650121e73a..b9dc436c6ed 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -68,7 +68,7 @@ func (c *goBinaryCataloger) parseGoBinary(resolver file.Resolver, _ *generic.Env pkgs = append(pkgs, goPkgs...) } - for key, _ := range compilerVersions { + for key := range compilerVersions { pkg := newGoStdLib(key) pkg.SetID() pkgs = append(pkgs, pkg) diff --git a/test/integration/go_compiler_detection_test.go b/test/integration/go_compiler_detection_test.go new file mode 100644 index 00000000000..df990c912c5 --- /dev/null +++ b/test/integration/go_compiler_detection_test.go @@ -0,0 +1,36 @@ +package integration + +import ( + "testing" + + "github.com/anchore/syft/syft/source" +) + +func TestGolangCompilerDetection(t *testing.T) { + tests := []struct { + name string + image string + expectedCompilers []string + }{ + { + name: "syft can detect a single golang compiler given the golang base image", + image: "image-golang-compiler", + expectedCompilers: []string{"1.18.10"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + sbom, _ := catalogFixtureImage(t, tt.image, source.SquashedScope, nil) + packages := sbom.Artifacts.Packages.PackagesByName("Golang Standard Library") + foundCompilerVersions := make(map[string]struct{}) + for _, pkg := range packages { + foundCompilerVersions[pkg.Version] = struct{}{} + } + for _, expectedCompiler := range tt.expectedCompilers { + if _, ok := foundCompilerVersions[expectedCompiler]; !ok { + t.Fatalf("expected %s version not found in found compilers: %v", expectedCompiler, foundCompilerVersions) + } + } + }) + } +} diff --git a/test/integration/test-fixtures/image-golang-compiler/Dockerfile b/test/integration/test-fixtures/image-golang-compiler/Dockerfile new file mode 100644 index 00000000000..2d8e6bbdce5 --- /dev/null +++ b/test/integration/test-fixtures/image-golang-compiler/Dockerfile @@ -0,0 +1 @@ +FROM golang:1.18.10-alpine \ No newline at end of file From d66e4ccb65682fb72d79fd71390375ff238b2914 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Wed, 4 Oct 2023 12:24:02 -0400 Subject: [PATCH 04/11] test: update integration tests to cover compiler detection Signed-off-by: Christopher Phillips --- .../integration/go_compiler_detection_test.go | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/integration/go_compiler_detection_test.go b/test/integration/go_compiler_detection_test.go index df990c912c5..7e51d26f111 100644 --- a/test/integration/go_compiler_detection_test.go +++ b/test/integration/go_compiler_detection_test.go @@ -3,6 +3,7 @@ package integration import ( "testing" + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/source" ) @@ -11,24 +12,49 @@ func TestGolangCompilerDetection(t *testing.T) { name string image string expectedCompilers []string + expectedCPE []cpe.CPE + expectedPURL []string }{ { name: "syft can detect a single golang compiler given the golang base image", image: "image-golang-compiler", expectedCompilers: []string{"1.18.10"}, + expectedCPE: []cpe.CPE{cpe.Must("cpe:2.3:a:golang:go:1.18.10:-:*:*:*:*:*:*")}, + expectedPURL: []string{"pkg:golang/stdlib@1.18.10"}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sbom, _ := catalogFixtureImage(t, tt.image, source.SquashedScope, nil) packages := sbom.Artifacts.Packages.PackagesByName("Golang Standard Library") + foundCompilerVersions := make(map[string]struct{}) + foundCPE := make(map[cpe.CPE]struct{}) + foundPURL := make(map[string]struct{}) + for _, pkg := range packages { foundCompilerVersions[pkg.Version] = struct{}{} + foundPURL[pkg.PURL] = struct{}{} + for _, cpe := range pkg.CPEs { + foundCPE[cpe] = struct{}{} + } } + for _, expectedCompiler := range tt.expectedCompilers { if _, ok := foundCompilerVersions[expectedCompiler]; !ok { - t.Fatalf("expected %s version not found in found compilers: %v", expectedCompiler, foundCompilerVersions) + t.Fatalf("expected %s version; not found in found compilers: %v", expectedCompiler, foundCompilerVersions) + } + } + + for _, expectedPURL := range tt.expectedPURL { + if _, ok := foundPURL[expectedPURL]; !ok { + t.Fatalf("expected %s purl; not found in found purl: %v", expectedPURL, expectedPURLs) + } + } + + for _, expectedCPE := range tt.expectedCPE { + if _, ok := foundCPE[expectedCPE]; !ok { + t.Fatalf("expected %s version; not found in found cpe: %v", expectedCPE, expectedCPE) } } }) From 1ca17dd8c11230475509bfe2b0eb649c3abd131c Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Wed, 4 Oct 2023 12:28:21 -0400 Subject: [PATCH 05/11] fix: remove incidental wip commit Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/catalog.go | 1 + syft/pkg/cataloger/golang/package.go | 10 ---------- syft/pkg/cataloger/golang/parse_go_binary.go | 7 ++++--- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/syft/pkg/cataloger/catalog.go b/syft/pkg/cataloger/catalog.go index 841a4c28baf..840ea72b7aa 100644 --- a/syft/pkg/cataloger/catalog.go +++ b/syft/pkg/cataloger/catalog.go @@ -92,6 +92,7 @@ func runCataloger(cataloger pkg.Cataloger, resolver file.Resolver) (catalogerRes p.Language = pkg.LanguageFromPURL(p.PURL) } + // create file-to-package relationships for files owned by the package owningRelationships, err := packageFileOwnershipRelationships(p, resolver) if err != nil { log.WithFields("cataloger", cataloger.Name(), "package", p.Name, "error", err).Warnf("unable to create any package-file relationships") diff --git a/syft/pkg/cataloger/golang/package.go b/syft/pkg/cataloger/golang/package.go index 093d5de16fd..37942563bf3 100644 --- a/syft/pkg/cataloger/golang/package.go +++ b/syft/pkg/cataloger/golang/package.go @@ -1,14 +1,12 @@ package golang import ( - "fmt" "regexp" "runtime/debug" "strings" "github.com/anchore/packageurl-go" "github.com/anchore/syft/internal/log" - "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" ) @@ -47,14 +45,6 @@ func (c *goBinaryCataloger) newGoBinaryPackage(resolver file.Resolver, dep *debu return p } -func buildCompilerCPE(goVersion string) (cpe.CPE, error) { - cpe, err := cpe.New(fmt.Sprintf("pkg:golang/stdlib@%s", goVersion)) - if err != nil { - - } - return cpe, nil -} - func packageURL(moduleName, moduleVersion string) string { // source: https://github.com/package-url/purl-spec/blob/master/PURL-TYPES.rst#golang // note: "The version is often empty when a commit is not specified and should be the commit in most cases when available." diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index b9dc436c6ed..f82638bd2bb 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -16,6 +16,7 @@ import ( "golang.org/x/mod/module" "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" @@ -81,11 +82,11 @@ func newGoStdLib(version string) pkg.Package { cpes := make([]cpe.CPE, 0) compilerCPE, err := cpe.New(fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", version)) if err != nil { - /// + log.Warn("could not build cpe for given compiler version: %s", version) } else { cpes = append(cpes, compilerCPE) } - pkg := pkg.Package{ + goCompilerPkg := pkg.Package{ Name: "Golang Standard Library", Version: version, PURL: packageURL("stdlib", version), @@ -96,7 +97,7 @@ func newGoStdLib(version string) pkg.Package { Metadata: pkg.GolangBinMetadata{}, } - return pkg + return goCompilerPkg } func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location) pkg.Package { From 5617c1b654a06466a6ff46e133eabb28c1c9f970 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Thu, 5 Oct 2023 08:50:29 -0400 Subject: [PATCH 06/11] feat: lift compiler package creation to individual per location set Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/cataloger.go | 47 +++++++++++++++++++- syft/pkg/cataloger/golang/parse_go_binary.go | 46 +++---------------- 2 files changed, 52 insertions(+), 41 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index ee936da9682..eb57d8854fa 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -4,8 +4,13 @@ Package golang provides a concrete Cataloger implementation for go.mod files. package golang import ( + "fmt" + "strings" + "github.com/anchore/syft/internal" + "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/event/monitor" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" @@ -47,5 +52,45 @@ func (p *progressingCataloger) Name() string { func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, []artifact.Relationship, error) { defer p.progress.SetCompleted() - return p.cataloger.Catalog(resolver) + pkgs, relationships, err := p.cataloger.Catalog(resolver) + goCompilerPkgs := []pkg.Package{} + for _, p := range pkgs { + if mValue, ok := p.Metadata.(pkg.GolangBinMetadata); ok { + stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, p.Locations) + if stdLibPkg != nil { + goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) + } + } + } + pkgs = append(pkgs, goCompilerPkgs...) + return pkgs, relationships, err +} +func newGoStdLib(version string, location file.LocationSet) *pkg.Package { + // for matching we need to strip the go prefix + // this can be preserved for metadata purposes + matchVersion := strings.TrimPrefix(version, "go") + cpes := make([]cpe.CPE, 0) + compilerCPE, err := cpe.New(fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", matchVersion)) + if err != nil { + log.Warn("could not build cpe for given compiler version: %s", version) + return nil + } + + cpes = append(cpes, compilerCPE) + goCompilerPkg := &pkg.Package{ + Name: "Golang Standard Library", + Version: version, + PURL: packageURL("stdlib", matchVersion), + CPEs: cpes, + Locations: location, + Language: pkg.Go, + Type: pkg.GoModulePkg, + MetadataType: pkg.GolangBinMetadataType, + Metadata: pkg.GolangBinMetadata{ + GoCompiledVersion: version, + }, + } + goCompilerPkg.SetID() + + return goCompilerPkg } diff --git a/syft/pkg/cataloger/golang/parse_go_binary.go b/syft/pkg/cataloger/golang/parse_go_binary.go index f82638bd2bb..5ccaaae3d60 100644 --- a/syft/pkg/cataloger/golang/parse_go_binary.go +++ b/syft/pkg/cataloger/golang/parse_go_binary.go @@ -16,9 +16,7 @@ import ( "golang.org/x/mod/module" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" - "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/file" "github.com/anchore/syft/syft/pkg" "github.com/anchore/syft/syft/pkg/cataloger/generic" @@ -61,45 +59,13 @@ func (c *goBinaryCataloger) parseGoBinary(resolver file.Resolver, _ *generic.Env mods := scanFile(unionReader, reader.RealPath) internal.CloseAndLogError(reader.ReadCloser, reader.RealPath) - compilerVersions := make(map[string]interface{}) - for _, mod := range mods { - goPkgs, goCompilerVersion := c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch) - compilerVersions[goCompilerVersion] = struct{}{} - pkgs = append(pkgs, goPkgs...) + pkgs = append(pkgs, c.buildGoPkgInfo(resolver, reader.Location, mod, mod.arch)...) } - for key := range compilerVersions { - pkg := newGoStdLib(key) - pkg.SetID() - pkgs = append(pkgs, pkg) - } return pkgs, nil, nil } -func newGoStdLib(version string) pkg.Package { - version = strings.TrimPrefix(version, "go") - cpes := make([]cpe.CPE, 0) - compilerCPE, err := cpe.New(fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", version)) - if err != nil { - log.Warn("could not build cpe for given compiler version: %s", version) - } else { - cpes = append(cpes, compilerCPE) - } - goCompilerPkg := pkg.Package{ - Name: "Golang Standard Library", - Version: version, - PURL: packageURL("stdlib", version), - CPEs: cpes, - Language: pkg.Go, - Type: pkg.GoModulePkg, - MetadataType: pkg.GolangBinMetadataType, - Metadata: pkg.GolangBinMetadata{}, - } - - return goCompilerPkg -} - func (c *goBinaryCataloger) makeGoMainPackage(resolver file.Resolver, mod *extendedBuildInfo, arch string, location file.Location) pkg.Package { gbs := getBuildSettings(mod.Settings) main := c.newGoBinaryPackage( @@ -254,10 +220,10 @@ func createMainModuleFromPath(path string) (mod debug.Module) { return } -func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string) (pkgs []pkg.Package, goCompilerVersion string) { - pkgs = make([]pkg.Package, 0) +func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file.Location, mod *extendedBuildInfo, arch string) []pkg.Package { + var pkgs []pkg.Package if mod == nil { - return pkgs, "" + return pkgs } var empty debug.Module @@ -285,11 +251,11 @@ func (c *goBinaryCataloger) buildGoPkgInfo(resolver file.Resolver, location file } if mod.Main == empty { - return pkgs, "" + return pkgs } main := c.makeGoMainPackage(resolver, mod, arch, location) pkgs = append(pkgs, main) - return pkgs, mod.GoVersion + return pkgs } From ea15f25658adb24317f9cd0de05d4233b6069990 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Thu, 5 Oct 2023 09:01:50 -0400 Subject: [PATCH 07/11] test: clean up integration tests adding new package and version Signed-off-by: Christopher Phillips --- test/integration/go_compiler_detection_test.go | 2 +- test/integration/regression_go_bin_scanner_arch_test.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/integration/go_compiler_detection_test.go b/test/integration/go_compiler_detection_test.go index 7e51d26f111..d244f394fb5 100644 --- a/test/integration/go_compiler_detection_test.go +++ b/test/integration/go_compiler_detection_test.go @@ -18,7 +18,7 @@ func TestGolangCompilerDetection(t *testing.T) { { name: "syft can detect a single golang compiler given the golang base image", image: "image-golang-compiler", - expectedCompilers: []string{"1.18.10"}, + expectedCompilers: []string{"go1.18.10"}, expectedCPE: []cpe.CPE{cpe.Must("cpe:2.3:a:golang:go:1.18.10:-:*:*:*:*:*:*")}, expectedPURL: []string{"pkg:golang/stdlib@1.18.10"}, }, diff --git a/test/integration/regression_go_bin_scanner_arch_test.go b/test/integration/regression_go_bin_scanner_arch_test.go index 8a51a9a77f2..d88ee6c7c6c 100644 --- a/test/integration/regression_go_bin_scanner_arch_test.go +++ b/test/integration/regression_go_bin_scanner_arch_test.go @@ -10,9 +10,9 @@ import ( func TestRegressionGoArchDiscovery(t *testing.T) { const ( - expectedELFPkg = 4 - expectedWINPkg = 4 - expectedMACOSPkg = 4 + expectedELFPkg = 5 + expectedWINPkg = 5 + expectedMACOSPkg = 5 ) // This is a regression test to make sure the way we detect go binary packages // stays consistent and reproducible as the tool chain evolves From 654145c1c25460c628b222c4a3efda7539aab1e5 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Thu, 5 Oct 2023 11:19:00 -0400 Subject: [PATCH 08/11] fix: unique compiler package per unique location set Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/cataloger.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index eb57d8854fa..cb8b87a110f 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -54,13 +54,24 @@ func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, [ defer p.progress.SetCompleted() pkgs, relationships, err := p.cataloger.Catalog(resolver) goCompilerPkgs := []pkg.Package{} - for _, p := range pkgs { - if mValue, ok := p.Metadata.(pkg.GolangBinMetadata); ok { - stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, p.Locations) - if stdLibPkg != nil { - goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) + locationMap := map[uint64]interface{}{} + for _, goPkg := range pkgs { + locationHash, err := goPkg.Locations.Hash() + if err != nil { + continue + } + + // we only need one synthetic compiler package per unique location set + if _, ok := locationMap[locationHash]; !ok { + if mValue, ok := goPkg.Metadata.(pkg.GolangBinMetadata); ok { + locationMap[locationHash] = true + stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations) + if stdLibPkg != nil { + goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) + } } } + } pkgs = append(pkgs, goCompilerPkgs...) return pkgs, relationships, err From 305c465110bfed5c8c2d80191bbd65ee67a8de8f Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Thu, 5 Oct 2023 11:32:32 -0400 Subject: [PATCH 09/11] chore: use location set implementation Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/cataloger.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index cb8b87a110f..ccdc01d9c7a 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -54,24 +54,20 @@ func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, [ defer p.progress.SetCompleted() pkgs, relationships, err := p.cataloger.Catalog(resolver) goCompilerPkgs := []pkg.Package{} - locationMap := map[uint64]interface{}{} + totalLocations := file.NewLocationSet() for _, goPkg := range pkgs { - locationHash, err := goPkg.Locations.Hash() - if err != nil { - continue - } - - // we only need one synthetic compiler package per unique location set - if _, ok := locationMap[locationHash]; !ok { - if mValue, ok := goPkg.Metadata.(pkg.GolangBinMetadata); ok { - locationMap[locationHash] = true - stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations) - if stdLibPkg != nil { - goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) + // go binary packages should only contain a single location + for _, location := range goPkg.Locations.ToSlice() { + if !totalLocations.Contains(location) { + if mValue, ok := goPkg.Metadata.(pkg.GolangBinMetadata); ok { + stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations) + if stdLibPkg != nil { + goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) + } + totalLocations.Add(location) } } } - } pkgs = append(pkgs, goCompilerPkgs...) return pkgs, relationships, err From 0d0d16c7132246810d7eeadf23647f87cdc57f07 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Thu, 5 Oct 2023 22:10:48 -0400 Subject: [PATCH 10/11] fix: update cpe generation and add unit test Signed-off-by: Christopher Phillips --- syft/pkg/cataloger/golang/cataloger.go | 59 +++++++++++++++------ syft/pkg/cataloger/golang/cataloger_test.go | 30 +++++++++++ 2 files changed, 72 insertions(+), 17 deletions(-) diff --git a/syft/pkg/cataloger/golang/cataloger.go b/syft/pkg/cataloger/golang/cataloger.go index ccdc01d9c7a..d1d117157a7 100644 --- a/syft/pkg/cataloger/golang/cataloger.go +++ b/syft/pkg/cataloger/golang/cataloger.go @@ -5,10 +5,10 @@ package golang import ( "fmt" + "regexp" "strings" "github.com/anchore/syft/internal" - "github.com/anchore/syft/internal/log" "github.com/anchore/syft/syft/artifact" "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/event/monitor" @@ -17,6 +17,8 @@ import ( "github.com/anchore/syft/syft/pkg/cataloger/generic" ) +var versionCandidateGroups = regexp.MustCompile(`(?P\d+(\.\d+)?(\.\d+)?)(?P\w*)`) + // NewGoModFileCataloger returns a new Go module cataloger object. func NewGoModFileCataloger(opts GoCatalogerOpts) pkg.Cataloger { c := goModCataloger{ @@ -56,14 +58,16 @@ func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, [ goCompilerPkgs := []pkg.Package{} totalLocations := file.NewLocationSet() for _, goPkg := range pkgs { + mValue, ok := goPkg.Metadata.(pkg.GolangBinMetadata) + if !ok { + continue + } // go binary packages should only contain a single location for _, location := range goPkg.Locations.ToSlice() { if !totalLocations.Contains(location) { - if mValue, ok := goPkg.Metadata.(pkg.GolangBinMetadata); ok { - stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations) - if stdLibPkg != nil { - goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) - } + stdLibPkg := newGoStdLib(mValue.GoCompiledVersion, goPkg.Locations) + if stdLibPkg != nil { + goCompilerPkgs = append(goCompilerPkgs, *stdLibPkg) totalLocations.Add(location) } } @@ -73,22 +77,15 @@ func (p *progressingCataloger) Catalog(resolver file.Resolver) ([]pkg.Package, [ return pkgs, relationships, err } func newGoStdLib(version string, location file.LocationSet) *pkg.Package { - // for matching we need to strip the go prefix - // this can be preserved for metadata purposes - matchVersion := strings.TrimPrefix(version, "go") - cpes := make([]cpe.CPE, 0) - compilerCPE, err := cpe.New(fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", matchVersion)) + stdlibCpe, err := generateStdlibCpe(version) if err != nil { - log.Warn("could not build cpe for given compiler version: %s", version) return nil } - - cpes = append(cpes, compilerCPE) goCompilerPkg := &pkg.Package{ - Name: "Golang Standard Library", + Name: "stdlib", Version: version, - PURL: packageURL("stdlib", matchVersion), - CPEs: cpes, + PURL: packageURL("stdlib", strings.TrimPrefix(version, "go")), + CPEs: []cpe.CPE{stdlibCpe}, Locations: location, Language: pkg.Go, Type: pkg.GoModulePkg, @@ -101,3 +98,31 @@ func newGoStdLib(version string, location file.LocationSet) *pkg.Package { return goCompilerPkg } + +func generateStdlibCpe(version string) (stdlibCpe cpe.CPE, err error) { + // GoCompiledVersion when pulled from a binary is prefixed by go + version = strings.TrimPrefix(version, "go") + + // we also need to trim starting from the first + to + // correctly extract potential rc candidate information for cpe generation + // ex: 2.0.0-rc.1+build.123 -> 2.0.0-rc.1; if no + is found then + is returned + after, _, found := strings.Cut("+", version) + if found { + version = after + } + + // extracting and + // https://regex101.com/r/985GsI/1 + captureGroups := internal.MatchNamedCaptureGroups(versionCandidateGroups, version) + vr, ok := captureGroups["version"] + if !ok || vr == "" { + return stdlibCpe, fmt.Errorf("could not match candidate version for: %s", version) + } + + cpeString := fmt.Sprintf("cpe:2.3:a:golang:go:%s:-:*:*:*:*:*:*", captureGroups["version"]) + if candidate, ok := captureGroups["candidate"]; ok && candidate != "" { + cpeString = fmt.Sprintf("cpe:2.3:a:golang:go:%s:%s:*:*:*:*:*:*", vr, candidate) + } + + return cpe.New(cpeString) +} diff --git a/syft/pkg/cataloger/golang/cataloger_test.go b/syft/pkg/cataloger/golang/cataloger_test.go index 7323e9fa804..b1e26ba3517 100644 --- a/syft/pkg/cataloger/golang/cataloger_test.go +++ b/syft/pkg/cataloger/golang/cataloger_test.go @@ -3,6 +3,9 @@ package golang import ( "testing" + "github.com/stretchr/testify/assert" + + "github.com/anchore/syft/syft/cpe" "github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest" ) @@ -56,3 +59,30 @@ func Test_Binary_Cataloger_Globs(t *testing.T) { }) } } + +func Test_Binary_Cataloger_Stdlib_Cpe(t *testing.T) { + tests := []struct { + name string + candidate string + want string + }{ + { + name: "generateStdlibCpe generates a cpe with a - for a major version", + candidate: "go1.21.0", + want: "cpe:2.3:a:golang:go:1.21.0:-:*:*:*:*:*:*", + }, + { + name: "generateStdlibCpe generates a cpe with an rc candidate for a major rc version", + candidate: "go1.21rc2", + want: "cpe:2.3:a:golang:go:1.21:rc2:*:*:*:*:*:*", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := generateStdlibCpe(tc.candidate) + assert.NoError(t, err, "expected no err; got %v", err) + assert.Equal(t, cpe.String(got), tc.want) + }) + } +} From add430fa9bf10077c439285d1bd3378a67a69fb0 Mon Sep 17 00:00:00 2001 From: Christopher Phillips Date: Fri, 6 Oct 2023 12:04:45 -0400 Subject: [PATCH 11/11] test: update integration test to use new name Signed-off-by: Christopher Phillips --- test/integration/go_compiler_detection_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/go_compiler_detection_test.go b/test/integration/go_compiler_detection_test.go index d244f394fb5..26994482379 100644 --- a/test/integration/go_compiler_detection_test.go +++ b/test/integration/go_compiler_detection_test.go @@ -26,7 +26,7 @@ func TestGolangCompilerDetection(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { sbom, _ := catalogFixtureImage(t, tt.image, source.SquashedScope, nil) - packages := sbom.Artifacts.Packages.PackagesByName("Golang Standard Library") + packages := sbom.Artifacts.Packages.PackagesByName("stdlib") foundCompilerVersions := make(map[string]struct{}) foundCPE := make(map[cpe.CPE]struct{})