Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Port javascript cataloger to new generic cataloger pattern #1308

Merged
merged 1 commit into from
Nov 2, 2022
Merged

Port javascript cataloger to new generic cataloger pattern #1308

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions syft/pkg/catalog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package pkg

import (
"sort"
"sync"

"github.com/jinzhu/copier"
Expand Down Expand Up @@ -199,24 +198,7 @@ func (c *Catalog) Sorted(types ...Type) (pkgs []Package) {
pkgs = append(pkgs, p)
}

sort.SliceStable(pkgs, func(i, j int) bool {
if pkgs[i].Name == pkgs[j].Name {
if pkgs[i].Version == pkgs[j].Version {
iLocations := pkgs[i].Locations.ToSlice()
jLocations := pkgs[j].Locations.ToSlice()
if pkgs[i].Type == pkgs[j].Type && len(iLocations) > 0 && len(jLocations) > 0 {
if iLocations[0].String() == jLocations[0].String() {
// compare IDs as a final fallback
return pkgs[i].ID() < pkgs[j].ID()
}
return iLocations[0].String() < jLocations[0].String()
}
return pkgs[i].Type < pkgs[j].Type
}
return pkgs[i].Version < pkgs[j].Version
}
return pkgs[i].Name < pkgs[j].Name
})
Sort(pkgs)

return pkgs
}
13 changes: 13 additions & 0 deletions syft/pkg/cataloger/internal/pkgtest/test_generic_parser.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,19 @@ func NewCatalogTester() *CatalogTester {
}
}

func (p *CatalogTester) FromDirectory(t *testing.T, path string) *CatalogTester {
t.Helper()

s, err := source.NewFromDirectory(path)
require.NoError(t, err)

resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)

p.resolver = resolver
return p
}

func (p *CatalogTester) FromFile(t *testing.T, path string) *CatalogTester {
t.Helper()

Expand Down
80 changes: 8 additions & 72 deletions syft/pkg/cataloger/javascript/cataloger.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,88 +4,24 @@ Package javascript provides a concrete Cataloger implementation for JavaScript e
package javascript

import (
"encoding/json"
"io"
"path"
"strings"

"github.com/anchore/syft/internal"
"github.com/anchore/syft/internal/log"
"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/common"
"github.com/anchore/syft/syft/pkg/cataloger/generic"
"github.com/anchore/syft/syft/source"
)

// NewJavascriptPackageCataloger returns a new JavaScript cataloger object based on detection of npm based packages.
func NewJavascriptPackageCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package.json": parsePackageJSON,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-package-cataloger")
func NewJavascriptPackageCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-package-cataloger").
WithParserByGlobs(parsePackageJSON, "**/package.json")
}

// NewJavascriptLockCataloger returns a new Javascript cataloger object base on package lock files.
func NewJavascriptLockCataloger() *common.GenericCataloger {
globParsers := map[string]common.ParserFn{
"**/package-lock.json": parsePackageLock,
"**/yarn.lock": parseYarnLock,
"**/pnpm-lock.yaml": parsePnpmLock,
}

return common.NewGenericCataloger(nil, globParsers, "javascript-lock-cataloger", addLicenses)
func NewJavascriptLockCataloger() *generic.Cataloger {
return generic.NewCataloger("javascript-lock-cataloger").
WithParserByGlobs(parsePackageLock, "**/package-lock.json").
WithParserByGlobs(parseYarnLock, "**/yarn.lock").
WithParserByGlobs(parsePnpmLock, "**/pnpm-lock.yaml")
}

func NewNodeBinaryCataloger() *generic.Cataloger {
return generic.NewCataloger("node-binary-cataloger").
WithParserByMimeTypes(parseNodeBinary, internal.ExecutableMIMETypeSet.List()...)
}

func addLicenses(resolver source.FileResolver, location source.Location, p *pkg.Package) error {
dir := path.Dir(location.RealPath)
pkgPath := []string{dir, "node_modules"}
pkgPath = append(pkgPath, strings.Split(p.Name, "/")...)
pkgPath = append(pkgPath, "package.json")
pkgFile := path.Join(pkgPath...)
locations, err := resolver.FilesByPath(pkgFile)
if err != nil {
log.Debugf("an error occurred attempting to read: %s - %+v", pkgFile, err)
return nil
}

if len(locations) == 0 {
return nil
}

for _, location := range locations {
contentReader, err := resolver.FileContentsByLocation(location)
if err != nil {
log.Debugf("error getting file content reader for %s: %v", pkgFile, err)
return nil
}

contents, err := io.ReadAll(contentReader)
if err != nil {
log.Debugf("error reading file contents for %s: %v", pkgFile, err)
return nil
}

var pkgJSON packageJSON
err = json.Unmarshal(contents, &pkgJSON)
if err != nil {
log.Debugf("error parsing %s: %v", pkgFile, err)
return nil
}

licenses, err := pkgJSON.licensesFromJSON()
if err != nil {
log.Debugf("error getting licenses from %s: %v", pkgFile, err)
return nil
}

p.Licenses = append(p.Licenses, licenses...)
}

return nil
}
174 changes: 97 additions & 77 deletions syft/pkg/cataloger/javascript/cataloger_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,100 +3,120 @@ package javascript
import (
"testing"

"github.com/stretchr/testify/require"

"github.com/anchore/syft/syft/pkg"
"github.com/anchore/syft/syft/pkg/cataloger/internal/pkgtest"
"github.com/anchore/syft/syft/source"
)

func Test_JavascriptCataloger(t *testing.T) {
expected := map[string]pkg.Package{
"@actions/core": {
Name: "@actions/core",
Version: "1.6.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
locationSet := source.NewLocationSet(source.NewLocation("package-lock.json"))
expectedPkgs := []pkg.Package{
{
Name: "@actions/core",
Version: "1.6.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/%40actions/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"wordwrap": {
Name: "wordwrap",
Version: "0.0.3",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "ansi-regex",
Version: "3.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"get-stdin": {
Name: "get-stdin",
Version: "5.0.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "cowsay",
Version: "1.4.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
},
"minimist": {
Name: "minimist",
Version: "0.0.10",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "get-stdin",
Version: "5.0.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"optimist": {
Name: "optimist",
Version: "0.6.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "is-fullwidth-code-point",
Version: "2.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"string-width": {
Name: "string-width",
Version: "2.1.1",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "minimist",
Version: "0.0.10",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-ansi": {
Name: "strip-ansi",
Version: "4.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "optimist",
Version: "0.6.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"strip-eof": {
Name: "wordwrap",
Version: "1.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "string-width",
Version: "2.1.1",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"ansi-regex": {
Name: "ansi-regex",
Version: "3.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-ansi",
Version: "4.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"is-fullwidth-code-point": {
Name: "is-fullwidth-code-point",
Version: "2.0.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
{
Name: "strip-eof",
Version: "1.0.0",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
"cowsay": {
Name: "cowsay",
Version: "1.4.0",
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
Licenses: []string{"MIT"},
{
Name: "wordwrap",
Version: "0.0.3",
FoundBy: "javascript-lock-cataloger",
PURL: "pkg:npm/[email protected]",
Locations: locationSet,
Language: pkg.JavaScript,
Type: pkg.NpmPkg,
},
}

s, err := source.NewFromDirectory("test-fixtures/pkg-lock")
require.NoError(t, err)

resolver, err := s.FileResolver(source.AllLayersScope)
require.NoError(t, err)

actual, _, err := NewJavascriptLockCataloger().Catalog(resolver)
if err != nil {
t.Fatalf("failed to parse package-lock.json: %+v", err)
}

var pkgs []*pkg.Package
for _, p := range actual {
p2 := p
pkgs = append(pkgs, &p2)
}
pkgtest.NewCatalogTester().
FromDirectory(t, "test-fixtures/pkg-lock").
Expects(expectedPkgs, nil).
TestCataloger(t, NewJavascriptLockCataloger())

assertPkgsEqual(t, pkgs, expected)
}
Loading