diff --git a/analyzer/config/config.go b/analyzer/config/config.go index d32690604..14b128549 100644 --- a/analyzer/config/config.go +++ b/analyzer/config/config.go @@ -28,18 +28,10 @@ type ScannerOption struct { } func (o *ScannerOption) Sort() { - sort.Slice(o.Namespaces, func(i, j int) bool { - return o.Namespaces[i] < o.Namespaces[j] - }) - sort.Slice(o.FilePatterns, func(i, j int) bool { - return o.FilePatterns[i] < o.FilePatterns[j] - }) - sort.Slice(o.PolicyPaths, func(i, j int) bool { - return o.PolicyPaths[i] < o.PolicyPaths[j] - }) - sort.Slice(o.DataPaths, func(i, j int) bool { - return o.DataPaths[i] < o.DataPaths[j] - }) + sort.Strings(o.Namespaces) + sort.Strings(o.FilePatterns) + sort.Strings(o.PolicyPaths) + sort.Strings(o.DataPaths) } func RegisterConfigAnalyzers(filePatterns []string) error { diff --git a/artifact/artifact.go b/artifact/artifact.go index 052cc6001..bb96b89ae 100644 --- a/artifact/artifact.go +++ b/artifact/artifact.go @@ -2,10 +2,28 @@ package artifact import ( "context" + "sort" + "github.com/aquasecurity/fanal/analyzer" + "github.com/aquasecurity/fanal/hook" "github.com/aquasecurity/fanal/types" ) +type Option struct { + DisabledAnalyzers []analyzer.Type + DisabledHooks []hook.Type + SkipFiles []string + SkipDirs []string +} + +func (o *Option) Sort() { + sort.Slice(o.DisabledAnalyzers, func(i, j int) bool { + return o.DisabledAnalyzers[i] < o.DisabledAnalyzers[j] + }) + sort.Strings(o.SkipFiles) + sort.Strings(o.SkipDirs) +} + type Artifact interface { Inspect(ctx context.Context) (reference types.ArtifactReference, err error) } diff --git a/artifact/image/image.go b/artifact/image/image.go index ef28cfd6e..e8dd25e40 100644 --- a/artifact/image/image.go +++ b/artifact/image/image.go @@ -49,36 +49,41 @@ var ( ) type Artifact struct { - image types.Image - cache cache.ArtifactCache - analyzer analyzer.Analyzer - hookManager hook.Manager - scanner scanner.Scanner + image types.Image + cache cache.ArtifactCache + walker walker.LayerTar + analyzer analyzer.Analyzer + hookManager hook.Manager + scanner scanner.Scanner + + artifactOption artifact.Option configScannerOption config.ScannerOption } -func NewArtifact(img types.Image, c cache.ArtifactCache, disabledAnalyzers []analyzer.Type, disabledHooks []hook.Type, - opt config.ScannerOption) (artifact.Artifact, error) { +func NewArtifact(img types.Image, c cache.ArtifactCache, artifactOpt artifact.Option, scannerOpt config.ScannerOption) (artifact.Artifact, error) { // Register config analyzers - if err := config.RegisterConfigAnalyzers(opt.FilePatterns); err != nil { + if err := config.RegisterConfigAnalyzers(scannerOpt.FilePatterns); err != nil { return nil, xerrors.Errorf("config scanner error: %w", err) } - s, err := scanner.New("", opt.Namespaces, opt.PolicyPaths, opt.DataPaths, opt.Trace) + s, err := scanner.New("", scannerOpt.Namespaces, scannerOpt.PolicyPaths, scannerOpt.DataPaths, scannerOpt.Trace) if err != nil { return nil, xerrors.Errorf("scanner error: %w", err) } - disabledAnalyzers = append(disabledAnalyzers, defaultDisabledAnalyzers...) - disabledHooks = append(disabledHooks, defaultDisabledHooks...) + disabledAnalyzers := append(artifactOpt.DisabledAnalyzers, defaultDisabledAnalyzers...) + disabledHooks := append(artifactOpt.DisabledHooks, defaultDisabledHooks...) return Artifact{ - image: img, - cache: c, - analyzer: analyzer.NewAnalyzer(disabledAnalyzers), - hookManager: hook.NewManager(disabledHooks), - scanner: s, - configScannerOption: opt, + image: img, + cache: c, + walker: walker.NewLayerTar(artifactOpt.SkipFiles, artifactOpt.SkipDirs), + analyzer: analyzer.NewAnalyzer(disabledAnalyzers), + hookManager: hook.NewManager(disabledHooks), + scanner: s, + + artifactOption: artifactOpt, + configScannerOption: scannerOpt, }, nil } @@ -142,7 +147,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) func (a Artifact) calcCacheKeys(imageID string, diffIDs []string) (string, []string, map[string]string, error) { // Pass an empty config scanner option so that the cache key can be the same, even when policies are updated. - imageKey, err := cache.CalcKey(imageID, a.analyzer.ImageConfigAnalyzerVersions(), nil, &config.ScannerOption{}) + imageKey, err := cache.CalcKey(imageID, a.analyzer.ImageConfigAnalyzerVersions(), nil, artifact.Option{}, config.ScannerOption{}) if err != nil { return "", nil, nil, err } @@ -151,7 +156,7 @@ func (a Artifact) calcCacheKeys(imageID string, diffIDs []string) (string, []str hookVersions := a.hookManager.Versions() var layerKeys []string for _, diffID := range diffIDs { - blobKey, err := cache.CalcKey(diffID, a.analyzer.AnalyzerVersions(), hookVersions, &a.configScannerOption) + blobKey, err := cache.CalcKey(diffID, a.analyzer.AnalyzerVersions(), hookVersions, a.artifactOption, a.configScannerOption) if err != nil { return "", nil, nil, err } @@ -218,7 +223,7 @@ func (a Artifact) inspectLayer(ctx context.Context, diffID string) (types.BlobIn result := new(analyzer.AnalysisResult) limit := semaphore.NewWeighted(parallel) - opqDirs, whFiles, err := walker.WalkLayerTar(r, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + opqDirs, whFiles, err := a.walker.Walk(r, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { if err = a.analyzer.AnalyzeFile(ctx, &wg, limit, result, "", filePath, info, opener); err != nil { return xerrors.Errorf("failed to analyze %s: %w", filePath, err) } diff --git a/artifact/image/image_test.go b/artifact/image/image_test.go index 99ed2d874..d482c8e7e 100644 --- a/artifact/image/image_test.go +++ b/artifact/image/image_test.go @@ -14,9 +14,9 @@ import ( "github.com/aquasecurity/fanal/analyzer" _ "github.com/aquasecurity/fanal/analyzer/all" "github.com/aquasecurity/fanal/analyzer/config" + "github.com/aquasecurity/fanal/artifact" image2 "github.com/aquasecurity/fanal/artifact/image" "github.com/aquasecurity/fanal/cache" - "github.com/aquasecurity/fanal/hook" _ "github.com/aquasecurity/fanal/hook/all" "github.com/aquasecurity/fanal/image" "github.com/aquasecurity/fanal/types" @@ -27,8 +27,7 @@ func TestArtifact_Inspect(t *testing.T) { tests := []struct { name string imagePath string - disableAnalyzers []analyzer.Type - disableHooks []hook.Type + artifactOpt artifact.Option missingBlobsExpectation cache.ArtifactCacheMissingBlobsExpectation putBlobExpectations []cache.ArtifactCachePutBlobExpectation putArtifactExpectations []cache.ArtifactCachePutArtifactExpectation @@ -40,18 +39,18 @@ func TestArtifact_Inspect(t *testing.T) { imagePath: "../../test/testdata/alpine-311.tar.gz", missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", - BlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", + BlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ MissingArtifact: true, - MissingBlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + MissingBlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, }, putBlobExpectations: []cache.ArtifactCachePutBlobExpectation{ { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29", + BlobID: "sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -90,7 +89,7 @@ func TestArtifact_Inspect(t *testing.T) { putArtifactExpectations: []cache.ArtifactCachePutArtifactExpectation{ { Args: cache.ArtifactCachePutArtifactArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", ArtifactInfo: types.ArtifactInfo{ SchemaVersion: 1, Architecture: "amd64", @@ -104,8 +103,8 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: "../../test/testdata/alpine-311.tar.gz", Type: types.ArtifactContainerImage, - ID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", - BlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + ID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", + BlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, ImageMetadata: types.ImageMetadata{ ID: "sha256:a187dde48cd289ac374ad8539930628314bc581a481cdb41409c9289419ddb72", DiffIDs: []string{ @@ -150,27 +149,27 @@ func TestArtifact_Inspect(t *testing.T) { imagePath: "../../test/testdata/vuln-image.tar.gz", missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:b79b48d9023d85be348e989163bb5f78bf9d0a9e6004a656dba50841ff212693", + ArtifactID: "sha256:a646bb11d39c149d4aaf9b888233048e0848304e5abd75667ea6f21d540d800c", BlobIDs: []string{ - "sha256:0203a9d8a0a2ee515ae3d9d211372316791654f6d4b7f9f00a3c0906be054e93", - "sha256:02af4fbed4ea7e8021770ab6846c6183e763b45b0adb0215ef2377689d2d501b", - "sha256:ee974db343198546d609c78a705989ab295c06475d0ff0fd392f2a4970f040de", - "sha256:7ca1744c7455a336ec8c430c8f0e3091f6fac3b2130ce9a40f88e7c0becf47e6", + "sha256:d0763d5a489e2b7f6e07a7f0892753a8d4d2c5836bbfb514d14f7626f443296f", + "sha256:6ffc82d2ae1b1e55510ef38980c8bca29bec307871ca20f142ded27a326fadc0", + "sha256:c84807e83484bed5c3cdc6ebf9463872c9d884fc74d1af54063f4ed0250921df", + "sha256:e9553dfdbedfca750f977af770147483f2e4480e0fb45478bd3747b171028a99", }, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ MissingBlobIDs: []string{ - "sha256:0203a9d8a0a2ee515ae3d9d211372316791654f6d4b7f9f00a3c0906be054e93", - "sha256:02af4fbed4ea7e8021770ab6846c6183e763b45b0adb0215ef2377689d2d501b", - "sha256:ee974db343198546d609c78a705989ab295c06475d0ff0fd392f2a4970f040de", - "sha256:7ca1744c7455a336ec8c430c8f0e3091f6fac3b2130ce9a40f88e7c0becf47e6", + "sha256:d0763d5a489e2b7f6e07a7f0892753a8d4d2c5836bbfb514d14f7626f443296f", + "sha256:6ffc82d2ae1b1e55510ef38980c8bca29bec307871ca20f142ded27a326fadc0", + "sha256:c84807e83484bed5c3cdc6ebf9463872c9d884fc74d1af54063f4ed0250921df", + "sha256:e9553dfdbedfca750f977af770147483f2e4480e0fb45478bd3747b171028a99", }, }, }, putBlobExpectations: []cache.ArtifactCachePutBlobExpectation{ { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:0203a9d8a0a2ee515ae3d9d211372316791654f6d4b7f9f00a3c0906be054e93", + BlobID: "sha256:d0763d5a489e2b7f6e07a7f0892753a8d4d2c5836bbfb514d14f7626f443296f", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -204,7 +203,7 @@ func TestArtifact_Inspect(t *testing.T) { }, { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:02af4fbed4ea7e8021770ab6846c6183e763b45b0adb0215ef2377689d2d501b", + BlobID: "sha256:6ffc82d2ae1b1e55510ef38980c8bca29bec307871ca20f142ded27a326fadc0", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -234,7 +233,7 @@ func TestArtifact_Inspect(t *testing.T) { }, { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:ee974db343198546d609c78a705989ab295c06475d0ff0fd392f2a4970f040de", + BlobID: "sha256:c84807e83484bed5c3cdc6ebf9463872c9d884fc74d1af54063f4ed0250921df", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -264,7 +263,7 @@ func TestArtifact_Inspect(t *testing.T) { { // Gemfile.lock will not be scanned. Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:7ca1744c7455a336ec8c430c8f0e3091f6fac3b2130ce9a40f88e7c0becf47e6", + BlobID: "sha256:e9553dfdbedfca750f977af770147483f2e4480e0fb45478bd3747b171028a99", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -277,12 +276,12 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: "../../test/testdata/vuln-image.tar.gz", Type: types.ArtifactContainerImage, - ID: "sha256:b79b48d9023d85be348e989163bb5f78bf9d0a9e6004a656dba50841ff212693", + ID: "sha256:a646bb11d39c149d4aaf9b888233048e0848304e5abd75667ea6f21d540d800c", BlobIDs: []string{ - "sha256:0203a9d8a0a2ee515ae3d9d211372316791654f6d4b7f9f00a3c0906be054e93", - "sha256:02af4fbed4ea7e8021770ab6846c6183e763b45b0adb0215ef2377689d2d501b", - "sha256:ee974db343198546d609c78a705989ab295c06475d0ff0fd392f2a4970f040de", - "sha256:7ca1744c7455a336ec8c430c8f0e3091f6fac3b2130ce9a40f88e7c0becf47e6", + "sha256:d0763d5a489e2b7f6e07a7f0892753a8d4d2c5836bbfb514d14f7626f443296f", + "sha256:6ffc82d2ae1b1e55510ef38980c8bca29bec307871ca20f142ded27a326fadc0", + "sha256:c84807e83484bed5c3cdc6ebf9463872c9d884fc74d1af54063f4ed0250921df", + "sha256:e9553dfdbedfca750f977af770147483f2e4480e0fb45478bd3747b171028a99", }, ImageMetadata: types.ImageMetadata{ ID: "sha256:58701fd185bda36cab0557bb6438661831267aa4a9e0b54211c4d5317a48aff4", @@ -353,32 +352,34 @@ func TestArtifact_Inspect(t *testing.T) { }, }, { - name: "happy path: disable analyzers", - imagePath: "../../test/testdata/vuln-image.tar.gz", - disableAnalyzers: []analyzer.Type{analyzer.TypeDebian, analyzer.TypeDpkg, analyzer.TypeComposer, analyzer.TypeBundler}, + name: "happy path: disable analyzers", + imagePath: "../../test/testdata/vuln-image.tar.gz", + artifactOpt: artifact.Option{ + DisabledAnalyzers: []analyzer.Type{analyzer.TypeDebian, analyzer.TypeDpkg, analyzer.TypeComposer, analyzer.TypeBundler}, + }, missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:b79b48d9023d85be348e989163bb5f78bf9d0a9e6004a656dba50841ff212693", + ArtifactID: "sha256:a646bb11d39c149d4aaf9b888233048e0848304e5abd75667ea6f21d540d800c", BlobIDs: []string{ - "sha256:f964ce5cb9cb1721515e117ce12e9166e71a251f58c34b4752ffe19b9bf7846e", - "sha256:9629d21a44aafadb938fe26a48d9eb75a92a590f43d38ade3da44c965472dc39", - "sha256:9811da603662e8806d723b2bafc1fcbae8292596f3ed78a49abfed7da0c26559", - "sha256:5437593858561686d969e14d2fa4b7badf3a7d82f4cd688d0c067b94b505f131", + "sha256:0f7226cd425bd908a02e290e4b76ebd63126decbaefbb320b72092e7f8d9e26c", + "sha256:48e480688081001f0b6d727535678b5dccc17c6fd11ee6d0f35a46cfaf0944b7", + "sha256:8ae034976a19798bb63db3cfd3ff7c9983b919b74a31e14f657667370e18873e", + "sha256:92c42af73d54b46304e282cc124eb26dc76783937351b540d3c21bc1c7146e7a", }, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ MissingBlobIDs: []string{ - "sha256:f964ce5cb9cb1721515e117ce12e9166e71a251f58c34b4752ffe19b9bf7846e", - "sha256:9629d21a44aafadb938fe26a48d9eb75a92a590f43d38ade3da44c965472dc39", - "sha256:9811da603662e8806d723b2bafc1fcbae8292596f3ed78a49abfed7da0c26559", - "sha256:5437593858561686d969e14d2fa4b7badf3a7d82f4cd688d0c067b94b505f131", + "sha256:0f7226cd425bd908a02e290e4b76ebd63126decbaefbb320b72092e7f8d9e26c", + "sha256:48e480688081001f0b6d727535678b5dccc17c6fd11ee6d0f35a46cfaf0944b7", + "sha256:8ae034976a19798bb63db3cfd3ff7c9983b919b74a31e14f657667370e18873e", + "sha256:92c42af73d54b46304e282cc124eb26dc76783937351b540d3c21bc1c7146e7a", }, }, }, putBlobExpectations: []cache.ArtifactCachePutBlobExpectation{ { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:f964ce5cb9cb1721515e117ce12e9166e71a251f58c34b4752ffe19b9bf7846e", + BlobID: "sha256:0f7226cd425bd908a02e290e4b76ebd63126decbaefbb320b72092e7f8d9e26c", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -388,7 +389,7 @@ func TestArtifact_Inspect(t *testing.T) { }, { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:9629d21a44aafadb938fe26a48d9eb75a92a590f43d38ade3da44c965472dc39", + BlobID: "sha256:48e480688081001f0b6d727535678b5dccc17c6fd11ee6d0f35a46cfaf0944b7", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -398,7 +399,7 @@ func TestArtifact_Inspect(t *testing.T) { }, { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:9811da603662e8806d723b2bafc1fcbae8292596f3ed78a49abfed7da0c26559", + BlobID: "sha256:8ae034976a19798bb63db3cfd3ff7c9983b919b74a31e14f657667370e18873e", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -409,7 +410,7 @@ func TestArtifact_Inspect(t *testing.T) { }, { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:5437593858561686d969e14d2fa4b7badf3a7d82f4cd688d0c067b94b505f131", + BlobID: "sha256:92c42af73d54b46304e282cc124eb26dc76783937351b540d3c21bc1c7146e7a", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -422,12 +423,12 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: "../../test/testdata/vuln-image.tar.gz", Type: types.ArtifactContainerImage, - ID: "sha256:b79b48d9023d85be348e989163bb5f78bf9d0a9e6004a656dba50841ff212693", + ID: "sha256:a646bb11d39c149d4aaf9b888233048e0848304e5abd75667ea6f21d540d800c", BlobIDs: []string{ - "sha256:f964ce5cb9cb1721515e117ce12e9166e71a251f58c34b4752ffe19b9bf7846e", - "sha256:9629d21a44aafadb938fe26a48d9eb75a92a590f43d38ade3da44c965472dc39", - "sha256:9811da603662e8806d723b2bafc1fcbae8292596f3ed78a49abfed7da0c26559", - "sha256:5437593858561686d969e14d2fa4b7badf3a7d82f4cd688d0c067b94b505f131", + "sha256:0f7226cd425bd908a02e290e4b76ebd63126decbaefbb320b72092e7f8d9e26c", + "sha256:48e480688081001f0b6d727535678b5dccc17c6fd11ee6d0f35a46cfaf0944b7", + "sha256:8ae034976a19798bb63db3cfd3ff7c9983b919b74a31e14f657667370e18873e", + "sha256:92c42af73d54b46304e282cc124eb26dc76783937351b540d3c21bc1c7146e7a", }, ImageMetadata: types.ImageMetadata{ ID: "sha256:58701fd185bda36cab0557bb6438661831267aa4a9e0b54211c4d5317a48aff4", @@ -492,8 +493,8 @@ func TestArtifact_Inspect(t *testing.T) { imagePath: "../../test/testdata/alpine-311.tar.gz", missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", - BlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", + BlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ Err: xerrors.New("MissingBlobs failed"), @@ -506,17 +507,17 @@ func TestArtifact_Inspect(t *testing.T) { imagePath: "../../test/testdata/alpine-311.tar.gz", missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", - BlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", + BlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ - MissingBlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + MissingBlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, }, putBlobExpectations: []cache.ArtifactCachePutBlobExpectation{ { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29", + BlobID: "sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -561,18 +562,18 @@ func TestArtifact_Inspect(t *testing.T) { imagePath: "../../test/testdata/alpine-311.tar.gz", missingBlobsExpectation: cache.ArtifactCacheMissingBlobsExpectation{ Args: cache.ArtifactCacheMissingBlobsArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", - BlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", + BlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, Returns: cache.ArtifactCacheMissingBlobsReturns{ MissingArtifact: true, - MissingBlobIDs: []string{"sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29"}, + MissingBlobIDs: []string{"sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9"}, }, }, putBlobExpectations: []cache.ArtifactCachePutBlobExpectation{ { Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:609d3e92df961c3a66d0bdb61f07946aac7614489ef96d6dee7c2e833fbc5f29", + BlobID: "sha256:6707264edbc3e72ae3def1158536888e34ea8f77fc697a33022194e378af36c9", BlobInfo: types.BlobInfo{ SchemaVersion: 1, Digest: "", @@ -611,7 +612,7 @@ func TestArtifact_Inspect(t *testing.T) { putArtifactExpectations: []cache.ArtifactCachePutArtifactExpectation{ { Args: cache.ArtifactCachePutArtifactArgs{ - ArtifactID: "sha256:59c4082ceb491faefd44cf9a006dd24c8f57b44b438f081251c90ea1367ca043", + ArtifactID: "sha256:059741cfbdc039e88e337d621e57e03e99b0e0a75df32f2027ebef13f839af65", ArtifactInfo: types.ArtifactInfo{ SchemaVersion: 1, Architecture: "amd64", @@ -638,7 +639,7 @@ func TestArtifact_Inspect(t *testing.T) { img, err := image.NewArchiveImage(tt.imagePath) require.NoError(t, err) - a, err := image2.NewArtifact(img, mockCache, tt.disableAnalyzers, tt.disableHooks, config.ScannerOption{}) + a, err := image2.NewArtifact(img, mockCache, tt.artifactOpt, config.ScannerOption{}) require.NoError(t, err) got, err := a.Inspect(context.Background()) diff --git a/artifact/local/fs.go b/artifact/local/fs.go index b79800167..11562f667 100644 --- a/artifact/local/fs.go +++ b/artifact/local/fs.go @@ -29,33 +29,38 @@ const ( ) type Artifact struct { - dir string - cache cache.ArtifactCache - analyzer analyzer.Analyzer - hookManager hook.Manager - scanner scanner.Scanner + dir string + cache cache.ArtifactCache + walker walker.Dir + analyzer analyzer.Analyzer + hookManager hook.Manager + scanner scanner.Scanner + + artifactOption artifact.Option configScannerOption config.ScannerOption } -func NewArtifact(dir string, c cache.ArtifactCache, disabledAnalyzers []analyzer.Type, disableHooks []hook.Type, - opt config.ScannerOption) (artifact.Artifact, error) { +func NewArtifact(dir string, c cache.ArtifactCache, artifactOpt artifact.Option, scannerOpt config.ScannerOption) (artifact.Artifact, error) { // Register config analyzers - if err := config.RegisterConfigAnalyzers(opt.FilePatterns); err != nil { + if err := config.RegisterConfigAnalyzers(scannerOpt.FilePatterns); err != nil { return nil, xerrors.Errorf("config analyzer error: %w", err) } - s, err := scanner.New(dir, opt.Namespaces, opt.PolicyPaths, opt.DataPaths, opt.Trace) + s, err := scanner.New(dir, scannerOpt.Namespaces, scannerOpt.PolicyPaths, scannerOpt.DataPaths, scannerOpt.Trace) if err != nil { return nil, xerrors.Errorf("scanner error: %w", err) } return Artifact{ - dir: dir, - cache: c, - analyzer: analyzer.NewAnalyzer(disabledAnalyzers), - hookManager: hook.NewManager(disableHooks), - scanner: s, - configScannerOption: opt, + dir: dir, + cache: c, + walker: walker.NewDir(artifactOpt.SkipFiles, artifactOpt.SkipDirs), + analyzer: analyzer.NewAnalyzer(artifactOpt.DisabledAnalyzers), + hookManager: hook.NewManager(artifactOpt.DisabledHooks), + scanner: s, + + artifactOption: artifactOpt, + configScannerOption: scannerOpt, }, nil } @@ -64,7 +69,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) result := new(analyzer.AnalysisResult) limit := semaphore.NewWeighted(parallel) - err := walker.WalkDir(a.dir, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + err := a.walker.Walk(a.dir, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { // For exported rootfs (e.g. images/alpine/etc/alpine-release) filePath, err := filepath.Rel(a.dir, filePath) if err != nil { @@ -113,7 +118,8 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) d := digest.NewDigest(digest.SHA256, h) diffID := d.String() blobInfo.DiffID = diffID - cacheKey, err := cache.CalcKey(diffID, a.analyzer.AnalyzerVersions(), a.hookManager.Versions(), &a.configScannerOption) + cacheKey, err := cache.CalcKey(diffID, a.analyzer.AnalyzerVersions(), a.hookManager.Versions(), + a.artifactOption, a.configScannerOption) if err != nil { return types.ArtifactReference{}, xerrors.Errorf("cache key: %w", err) } diff --git a/artifact/local/fs_test.go b/artifact/local/fs_test.go index e21aae54c..30dae1e37 100644 --- a/artifact/local/fs_test.go +++ b/artifact/local/fs_test.go @@ -11,6 +11,7 @@ import ( "github.com/aquasecurity/fanal/analyzer" _ "github.com/aquasecurity/fanal/analyzer/all" "github.com/aquasecurity/fanal/analyzer/config" + "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/hook" _ "github.com/aquasecurity/fanal/hook/all" @@ -24,6 +25,7 @@ func TestArtifact_Inspect(t *testing.T) { tests := []struct { name string fields fields + artifactOpt artifact.Option scannerOpt config.ScannerOption disabledAnalyzers []analyzer.Type disabledHooks []hook.Type @@ -38,7 +40,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:b452b832b5b35485b18fc7d80baacfd4fc812cd3b07d7d02727330f7076a999c", + BlobID: "sha256:81daa14b721151ef1105433864c66913fe8e0eb7eb661981ad60e84330ff5e22", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, DiffID: "sha256:0f88c2f4a441514ebd105e81527af76af15bed17d91c17ba3637397f8c4f1925", @@ -61,9 +63,9 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: "host", Type: types.ArtifactFilesystem, - ID: "sha256:b452b832b5b35485b18fc7d80baacfd4fc812cd3b07d7d02727330f7076a999c", + ID: "sha256:81daa14b721151ef1105433864c66913fe8e0eb7eb661981ad60e84330ff5e22", BlobIDs: []string{ - "sha256:b452b832b5b35485b18fc7d80baacfd4fc812cd3b07d7d02727330f7076a999c", + "sha256:81daa14b721151ef1105433864c66913fe8e0eb7eb661981ad60e84330ff5e22", }, }, }, @@ -72,10 +74,12 @@ func TestArtifact_Inspect(t *testing.T) { fields: fields{ dir: "./testdata", }, - disabledAnalyzers: []analyzer.Type{analyzer.TypeAlpine, analyzer.TypeApk}, + artifactOpt: artifact.Option{ + DisabledAnalyzers: []analyzer.Type{analyzer.TypeAlpine, analyzer.TypeApk}, + }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:0c144e61b3867b1b6ef5d58620f08e9434fbd30e731094e92feb25060dee03b4", + BlobID: "sha256:9f59d064662e306b70aaa32db99bd51ade23da5bd983aceb7b2b916fdb43bceb", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, DiffID: "sha256:3404e98968ad338dc60ef74c0dd5bdd893478415cd2296b0c265a5650b3ae4d6", @@ -86,9 +90,9 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: "host", Type: types.ArtifactFilesystem, - ID: "sha256:0c144e61b3867b1b6ef5d58620f08e9434fbd30e731094e92feb25060dee03b4", + ID: "sha256:9f59d064662e306b70aaa32db99bd51ade23da5bd983aceb7b2b916fdb43bceb", BlobIDs: []string{ - "sha256:0c144e61b3867b1b6ef5d58620f08e9434fbd30e731094e92feb25060dee03b4", + "sha256:9f59d064662e306b70aaa32db99bd51ade23da5bd983aceb7b2b916fdb43bceb", }, }, }, @@ -99,7 +103,7 @@ func TestArtifact_Inspect(t *testing.T) { }, putBlobExpectation: cache.ArtifactCachePutBlobExpectation{ Args: cache.ArtifactCachePutBlobArgs{ - BlobID: "sha256:b452b832b5b35485b18fc7d80baacfd4fc812cd3b07d7d02727330f7076a999c", + BlobID: "sha256:81daa14b721151ef1105433864c66913fe8e0eb7eb661981ad60e84330ff5e22", BlobInfo: types.BlobInfo{ SchemaVersion: types.BlobJSONSchemaVersion, DiffID: "sha256:0f88c2f4a441514ebd105e81527af76af15bed17d91c17ba3637397f8c4f1925", @@ -136,7 +140,7 @@ func TestArtifact_Inspect(t *testing.T) { c := new(cache.MockArtifactCache) c.ApplyPutBlobExpectation(tt.putBlobExpectation) - a, err := NewArtifact(tt.fields.dir, c, tt.disabledAnalyzers, tt.disabledHooks, tt.scannerOpt) + a, err := NewArtifact(tt.fields.dir, c, tt.artifactOpt, tt.scannerOpt) require.NoError(t, err) got, err := a.Inspect(context.Background()) diff --git a/artifact/remote/git.go b/artifact/remote/git.go index 00314669b..c317e384a 100644 --- a/artifact/remote/git.go +++ b/artifact/remote/git.go @@ -14,7 +14,6 @@ import ( "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/artifact/local" "github.com/aquasecurity/fanal/cache" - "github.com/aquasecurity/fanal/hook" "github.com/aquasecurity/fanal/types" ) @@ -37,8 +36,7 @@ type Artifact struct { local artifact.Artifact } -func NewArtifact(rawurl string, c cache.ArtifactCache, disabledAnalyzers []analyzer.Type, disabledHooks []hook.Type, - opt config.ScannerOption) ( +func NewArtifact(rawurl string, c cache.ArtifactCache, artifactOpt artifact.Option, scannerOpt config.ScannerOption) ( artifact.Artifact, func(), error) { cleanup := func() {} @@ -65,9 +63,9 @@ func NewArtifact(rawurl string, c cache.ArtifactCache, disabledAnalyzers []analy _ = os.RemoveAll(tmpDir) } - disabledAnalyzers = append(disabledAnalyzers, defaultDisabledAnalyzers...) + artifactOpt.DisabledAnalyzers = append(artifactOpt.DisabledAnalyzers, defaultDisabledAnalyzers...) - art, err := local.NewArtifact(tmpDir, c, disabledAnalyzers, disabledHooks, opt) + art, err := local.NewArtifact(tmpDir, c, artifactOpt, scannerOpt) if err != nil { return nil, cleanup, xerrors.Errorf("fs artifact: %w", err) } @@ -93,7 +91,7 @@ func (a Artifact) Inspect(ctx context.Context) (types.ArtifactReference, error) func newURL(rawurl string) (*url.URL, error) { u, err := url.Parse(rawurl) if err != nil { - return nil, err + return nil, xerrors.Errorf("url parse error: %w", err) } // "https://" can be omitted // e.g. github.com/aquasecurity/fanal diff --git a/artifact/remote/git_test.go b/artifact/remote/git_test.go index 055ced197..61f25ffc0 100644 --- a/artifact/remote/git_test.go +++ b/artifact/remote/git_test.go @@ -10,6 +10,7 @@ import ( "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer/config" + "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/types" ) @@ -70,7 +71,7 @@ func TestNewArtifact(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, cleanup, err := NewArtifact(tt.args.rawurl, tt.args.c, nil, nil, config.ScannerOption{}) + _, cleanup, err := NewArtifact(tt.args.rawurl, tt.args.c, artifact.Option{}, config.ScannerOption{}) assert.Equal(t, tt.wantErr, err != nil) defer cleanup() }) @@ -94,9 +95,9 @@ func TestArtifact_Inspect(t *testing.T) { want: types.ArtifactReference{ Name: ts.URL + "/test.git", Type: types.ArtifactRemoteRepository, - ID: "sha256:2f64ff463b8e72dbb1010607351956b119817ad5376839f8c5e8624dbb804da6", + ID: "sha256:1b9a2e1c7079d3fa2e1ed8e3886cd271cbab400d6b5d5115d3b4f965f239b58e", BlobIDs: []string{ - "sha256:2f64ff463b8e72dbb1010607351956b119817ad5376839f8c5e8624dbb804da6", + "sha256:1b9a2e1c7079d3fa2e1ed8e3886cd271cbab400d6b5d5115d3b4f965f239b58e", }, }, }, @@ -107,7 +108,7 @@ func TestArtifact_Inspect(t *testing.T) { fsCache, err := cache.NewFSCache(t.TempDir()) require.NoError(t, err) - art, cleanup, err := NewArtifact(tt.rawurl, fsCache, nil, nil, config.ScannerOption{}) + art, cleanup, err := NewArtifact(tt.rawurl, fsCache, artifact.Option{}, config.ScannerOption{}) require.NoError(t, err) defer cleanup() diff --git a/cache/key.go b/cache/key.go index 94995a9ff..a1b37c14a 100644 --- a/cache/key.go +++ b/cache/key.go @@ -5,31 +5,36 @@ import ( "encoding/json" "fmt" + "github.com/aquasecurity/fanal/artifact" + "golang.org/x/mod/sumdb/dirhash" "golang.org/x/xerrors" "github.com/aquasecurity/fanal/analyzer/config" ) -func CalcKey(id string, analyzerVersions, hookVersions map[string]int, opt *config.ScannerOption) (string, error) { +func CalcKey(id string, analyzerVersions, hookVersions map[string]int, artifactOpt artifact.Option, scannerOpt config.ScannerOption) (string, error) { // Sort options for consistent results - opt.Sort() + artifactOpt.Sort() + scannerOpt.Sort() h := sha256.New() - if _, err := h.Write([]byte(id)); err != nil { - return "", xerrors.Errorf("sha256 error: %w", err) - } - - if err := json.NewEncoder(h).Encode(analyzerVersions); err != nil { - return "", xerrors.Errorf("json encode error: %w", err) - } + // Write ID, analyzer/hook versions, and skipped files/dirs + keyBase := struct { + ID string + AnalyzerVersions map[string]int + HookVersions map[string]int + SkipFiles []string + SkipDirs []string + }{id, analyzerVersions, hookVersions, artifactOpt.SkipFiles, artifactOpt.SkipDirs} - if err := json.NewEncoder(h).Encode(hookVersions); err != nil { + if err := json.NewEncoder(h).Encode(keyBase); err != nil { return "", xerrors.Errorf("json encode error: %w", err) } - for _, paths := range [][]string{opt.PolicyPaths, opt.DataPaths} { + // Write policy and data contents + for _, paths := range [][]string{scannerOpt.PolicyPaths, scannerOpt.DataPaths} { for _, p := range paths { s, err := dirhash.HashDir(p, "", dirhash.DefaultHash) if err != nil { diff --git a/cache/key_test.go b/cache/key_test.go index 2014890df..694ecbb30 100644 --- a/cache/key_test.go +++ b/cache/key_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer/config" + "github.com/aquasecurity/fanal/artifact" ) func TestCalcKey(t *testing.T) { @@ -14,6 +15,8 @@ func TestCalcKey(t *testing.T) { key string analyzerVersions map[string]int hookVersions map[string]int + skipFiles []string + skipDirs []string patterns []string policy []string data []string @@ -36,7 +39,7 @@ func TestCalcKey(t *testing.T) { "python-pkg": 1, }, }, - want: "sha256:dcdf828ab855918fc61688c8a9e27d011579601faef57122655d9e10128b2898", + want: "sha256:8060f9cc9ba29039785a7116ae874673ad7a6eab37170ee1375b4064a72343ae", }, { name: "with disabled analyzer", @@ -51,7 +54,7 @@ func TestCalcKey(t *testing.T) { "python-pkg": 1, }, }, - want: "sha256:796a061d4942fe2eb1a3559cf4a4cff2500eb2dc67d6cd4a1e5914250b6db994", + want: "sha256:e6a28d20a3a901377dcb836959c8ac268ec573735a5ba9c29112a1f6c5b1edd2", }, { name: "with empty slice file patterns", @@ -63,7 +66,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{}, }, - want: "sha256:0ecded9645d88f15c2372741abb3e52b4c4fc83ccad1df57a67e4b92b2da7200", + want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", }, { name: "with single empty string in file patterns", @@ -75,7 +78,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{""}, }, - want: "sha256:0ecded9645d88f15c2372741abb3e52b4c4fc83ccad1df57a67e4b92b2da7200", + want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", }, { name: "with single non empty string in file patterns", @@ -87,7 +90,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"test"}, }, - want: "sha256:0ecded9645d88f15c2372741abb3e52b4c4fc83ccad1df57a67e4b92b2da7200", + want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", }, { name: "with non empty followed by empty string in file patterns", @@ -99,7 +102,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"test", ""}, }, - want: "sha256:0ecded9645d88f15c2372741abb3e52b4c4fc83ccad1df57a67e4b92b2da7200", + want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", }, { name: "with non empty preceded by empty string in file patterns", @@ -111,7 +114,7 @@ func TestCalcKey(t *testing.T) { }, patterns: []string{"", "test"}, }, - want: "sha256:0ecded9645d88f15c2372741abb3e52b4c4fc83ccad1df57a67e4b92b2da7200", + want: "sha256:d69f13df33f4c159b4ea54c1967384782fcefb5e2a19af35f4cd6d2896e9285e", }, { name: "with policy", @@ -123,7 +126,21 @@ func TestCalcKey(t *testing.T) { }, policy: []string{"testdata"}, }, - want: "sha256:dd247485b0c153e3a438c70b06b1e711b78b8a5c08da3f0a8c76efdf3a4d3194", + want: "sha256:6865fae846fbe28290701e6828f51bf704dab746f06f3f7c0ec4d55c8d73da23", + }, + { + name: "skip files and dirs", + args: args{ + key: "sha256:5c534be56eca62e756ef2ef51523feda0f19cd7c15bb0c015e3d6e3ae090bf6e", + analyzerVersions: map[string]int{ + "alpine": 1, + "debian": 1, + }, + skipFiles: []string{"app/deployment.yaml"}, + skipDirs: []string{"usr/java"}, + policy: []string{"testdata"}, + }, + want: "sha256:7ddb3a93c2aa70c4a0b5d793110fa0f380318f9d7f795c4e28dfa59749c27416", }, { name: "with policy/non-existent dir", @@ -140,12 +157,16 @@ func TestCalcKey(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - opt := &config.ScannerOption{ + artifactOpt := artifact.Option{ + SkipFiles: tt.args.skipFiles, + SkipDirs: tt.args.skipDirs, + } + scannerOpt := config.ScannerOption{ FilePatterns: tt.args.patterns, PolicyPaths: tt.args.policy, DataPaths: tt.args.data, } - got, err := CalcKey(tt.args.key, tt.args.analyzerVersions, tt.args.hookVersions, opt) + got, err := CalcKey(tt.args.key, tt.args.analyzerVersions, tt.args.hookVersions, artifactOpt, scannerOpt) if tt.wantErr != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.wantErr) diff --git a/cmd/fanal/main.go b/cmd/fanal/main.go index 8efe5d10c..f2b10bec9 100644 --- a/cmd/fanal/main.go +++ b/cmd/fanal/main.go @@ -47,6 +47,14 @@ func run() (err error) { Name: "conf-policy", Usage: "policy paths", }, + &cli.StringSliceFlag{ + Name: "skip-files", + Usage: "skip files", + }, + &cli.StringSliceFlag{ + Name: "skip-dirs", + Usage: "skip dirs", + }, }, Action: globalOption(imageAction), }, @@ -70,6 +78,14 @@ func run() (err error) { Name: "policy", Usage: "policy paths", }, + &cli.StringSliceFlag{ + Name: "skip-files", + Usage: "skip files", + }, + &cli.StringSliceFlag{ + Name: "skip-dirs", + Usage: "skip dirs", + }, }, Action: globalOption(fsAction), }, @@ -127,9 +143,15 @@ func initializeCache(backend string) (cache.Cache, error) { } func imageAction(c *cli.Context, fsCache cache.Cache) error { - art, cleanup, err := imageArtifact(c.Context, c.Args().First(), fsCache, config.ScannerOption{ - PolicyPaths: c.StringSlice("conf-policy"), - }) + artifactOpt := artifact.Option{ + SkipFiles: c.StringSlice("skip-files"), + SkipDirs: c.StringSlice("skip-dirs"), + } + scannerOpt := config.ScannerOption{ + PolicyPaths: c.StringSlice("policy"), + } + + art, cleanup, err := imageArtifact(c.Context, c.Args().First(), fsCache, artifactOpt, scannerOpt) if err != nil { return err } @@ -146,10 +168,16 @@ func archiveAction(c *cli.Context, fsCache cache.Cache) error { } func fsAction(c *cli.Context, fsCache cache.Cache) error { - art, err := local.NewArtifact(c.Args().First(), fsCache, nil, nil, config.ScannerOption{ + artifactOpt := artifact.Option{ + SkipFiles: c.StringSlice("skip-files"), + SkipDirs: c.StringSlice("skip-dirs"), + } + scannerOpt := config.ScannerOption{ Namespaces: []string{"appshield"}, PolicyPaths: c.StringSlice("policy"), - }) + } + + art, err := local.NewArtifact(c.Args().First(), fsCache, artifactOpt, scannerOpt) if err != nil { return err } @@ -203,7 +231,8 @@ func inspect(ctx context.Context, art artifact.Artifact, c cache.LocalArtifactCa return nil } -func imageArtifact(ctx context.Context, imageName string, c cache.ArtifactCache, opt config.ScannerOption) (artifact.Artifact, func(), error) { +func imageArtifact(ctx context.Context, imageName string, c cache.ArtifactCache, + artifactOpt artifact.Option, scannerOpt config.ScannerOption) (artifact.Artifact, func(), error) { img, cleanup, err := image.NewDockerImage(ctx, imageName, types.DockerOption{ Timeout: 600 * time.Second, SkipPing: true, @@ -212,7 +241,7 @@ func imageArtifact(ctx context.Context, imageName string, c cache.ArtifactCache, return nil, func() {}, err } - art, err := aimage.NewArtifact(img, c, nil, nil, opt) + art, err := aimage.NewArtifact(img, c, artifactOpt, scannerOpt) if err != nil { return nil, func() {}, err } @@ -225,7 +254,7 @@ func archiveImageArtifact(imagePath string, c cache.ArtifactCache) (artifact.Art return nil, err } - art, err := aimage.NewArtifact(img, c, nil, nil, config.ScannerOption{}) + art, err := aimage.NewArtifact(img, c, artifact.Option{}, config.ScannerOption{}) if err != nil { return nil, err } @@ -233,5 +262,5 @@ func archiveImageArtifact(imagePath string, c cache.ArtifactCache) (artifact.Art } func remoteArtifact(dir string, c cache.ArtifactCache) (artifact.Artifact, func(), error) { - return remote.NewArtifact(dir, c, nil, nil, config.ScannerOption{}) + return remote.NewArtifact(dir, c, artifact.Option{}, config.ScannerOption{}) } diff --git a/external/config_scan.go b/external/config_scan.go index b3291902e..f3dddcb04 100644 --- a/external/config_scan.go +++ b/external/config_scan.go @@ -7,6 +7,7 @@ import ( "github.com/aquasecurity/fanal/analyzer" "github.com/aquasecurity/fanal/analyzer/config" "github.com/aquasecurity/fanal/applier" + "github.com/aquasecurity/fanal/artifact" "github.com/aquasecurity/fanal/artifact/local" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/types" @@ -35,7 +36,7 @@ func NewConfigScanner(cacheDir string, policyPaths, dataPaths, namespaces []stri } func (s ConfigScanner) Scan(dir string) ([]types.Misconfiguration, error) { - art, err := local.NewArtifact(dir, s.cache, nil, nil, config.ScannerOption{ + art, err := local.NewArtifact(dir, s.cache, artifact.Option{}, config.ScannerOption{ PolicyPaths: s.policyPaths, DataPaths: s.dataPaths, Namespaces: s.namespaces, diff --git a/test/integration/library_test.go b/test/integration/library_test.go index 6b5d21756..a1003d86b 100644 --- a/test/integration/library_test.go +++ b/test/integration/library_test.go @@ -126,7 +126,7 @@ func TestFanal_Library_DockerLessMode(t *testing.T) { require.NoError(t, err, tc.name) defer cleanup() - ar, err := aimage.NewArtifact(img, c, nil, nil, config.ScannerOption{}) + ar, err := aimage.NewArtifact(img, c, artifact.Option{}, config.ScannerOption{}) require.NoError(t, err) applier := applier.NewApplier(c) @@ -176,7 +176,7 @@ func TestFanal_Library_DockerMode(t *testing.T) { require.NoError(t, err, tc.name) defer cleanup() - ar, err := aimage.NewArtifact(img, c, nil, nil, config.ScannerOption{}) + ar, err := aimage.NewArtifact(img, c, artifact.Option{}, config.ScannerOption{}) require.NoError(t, err) applier := applier.NewApplier(c) @@ -221,7 +221,7 @@ func TestFanal_Library_TarMode(t *testing.T) { img, err := image.NewArchiveImage(tc.imageFile) require.NoError(t, err, tc.name) - ar, err := aimage.NewArtifact(img, c, nil, nil, config.ScannerOption{}) + ar, err := aimage.NewArtifact(img, c, artifact.Option{}, config.ScannerOption{}) require.NoError(t, err) applier := applier.NewApplier(c) diff --git a/test/integration/registry_test.go b/test/integration/registry_test.go index ea861ebff..be66cf9b2 100644 --- a/test/integration/registry_test.go +++ b/test/integration/registry_test.go @@ -1,3 +1,4 @@ +//go:build integration // +build integration package integration @@ -22,6 +23,7 @@ import ( _ "github.com/aquasecurity/fanal/analyzer/all" "github.com/aquasecurity/fanal/analyzer/config" "github.com/aquasecurity/fanal/applier" + "github.com/aquasecurity/fanal/artifact" aimage "github.com/aquasecurity/fanal/artifact/image" "github.com/aquasecurity/fanal/cache" "github.com/aquasecurity/fanal/image" @@ -202,7 +204,7 @@ func analyze(ctx context.Context, imageRef string, opt types.DockerOption) (*typ } defer cleanup() - ar, err := aimage.NewArtifact(img, c, nil, nil, config.ScannerOption{}) + ar, err := aimage.NewArtifact(img, c, artifact.Option{}, config.ScannerOption{}) if err != nil { return nil, err } diff --git a/types/artifact.go b/types/artifact.go index 1e6c8bbc4..4e647c6a7 100644 --- a/types/artifact.go +++ b/types/artifact.go @@ -50,7 +50,7 @@ type PackageInfo struct { } type LibraryInfo struct { - // Each package metadata have the file path, while a package from lock files do not have + // Each package metadata have the file path, while the package from lock files does not have FilePath string `json:",omitempty"` // Library holds package name, version, etc. diff --git a/walker/fs.go b/walker/fs.go index bd7845bb7..5bd089901 100644 --- a/walker/fs.go +++ b/walker/fs.go @@ -1,35 +1,55 @@ package walker import ( - "io/ioutil" "os" "path/filepath" - "sync" - - "github.com/saracen/walker" + swalker "github.com/saracen/walker" "golang.org/x/xerrors" ) -// WalkDir walks the file tree rooted at root, calling WalkFunc for each file or +type Dir struct { + walker +} + +func NewDir(skipFiles, skipDirs []string) Dir { + return Dir{ + walker: newWalker(skipFiles, skipDirs), + } +} + +// Walk walks the file tree rooted at root, calling WalkFunc for each file or // directory in the tree, including root, but a directory to be ignored will be skipped. -func WalkDir(root string, f WalkFunc) error { +func (w Dir) Walk(root string, fn WalkFunc) error { // walk function called for every path found walkFn := func(pathname string, fi os.FileInfo) error { - if !fi.Mode().IsRegular() { + pathname = filepath.Clean(pathname) + + if fi.IsDir() { + if w.shouldSkipDir(pathname) { + return filepath.SkipDir + } + return nil + } else if !fi.Mode().IsRegular() { + return nil + } else if w.shouldSkipFile(pathname) { return nil - } else if isIgnored(pathname) { - return filepath.SkipDir } - pathname = filepath.Clean(pathname) - if err := f(pathname, fi, fileOnceOpener(pathname)); err != nil { + + f, err := os.Open(pathname) + if err != nil { + return xerrors.Errorf("file open error (%s): %w", pathname, err) + } + defer f.Close() + + if err = fn(pathname, fi, w.fileOnceOpener(f)); err != nil { return xerrors.Errorf("failed to analyze file: %w", err) } return nil } // error function called for every error encountered - errorCallbackOption := walker.WithErrorCallback(func(pathname string, err error) error { + errorCallbackOption := swalker.WithErrorCallback(func(pathname string, err error) error { // ignore permission errors if os.IsPermission(err) { return nil @@ -40,25 +60,8 @@ func WalkDir(root string, f WalkFunc) error { // Multiple goroutines stat the filesystem concurrently. The provided // walkFn must be safe for concurrent use. - if err := walker.Walk(root, walkFn, errorCallbackOption); err != nil { - return err + if err := swalker.Walk(root, walkFn, errorCallbackOption); err != nil { + return xerrors.Errorf("walk error: %w", err) } return nil } - -// fileOnceOpener opens a file once and the content is shared so that some analyzers can use the same data -func fileOnceOpener(filePath string) func() ([]byte, error) { - var once sync.Once - var b []byte - var err error - - return func() ([]byte, error) { - once.Do(func() { - b, err = ioutil.ReadFile(filePath) - }) - if err != nil { - return nil, xerrors.Errorf("unable to read file: %w", err) - } - return b, nil - } -} diff --git a/walker/fs_test.go b/walker/fs_test.go index b3781d3c6..693751cdc 100644 --- a/walker/fs_test.go +++ b/walker/fs_test.go @@ -3,38 +3,86 @@ package walker_test import ( "errors" "os" + "strings" "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/aquasecurity/fanal/analyzer" "github.com/aquasecurity/fanal/walker" ) -func TestWalkDir(t *testing.T) { - // happy path - err := walker.WalkDir("testdata/fs", func(filePath string, info os.FileInfo, opener analyzer.Opener) error { - if info.IsDir() { - return nil - } - if filePath == "testdata/fs/bar" { - b, err := opener() - require.NoError(t, err) - assert.Equal(t, "bar", string(b)) - } else { - assert.Fail(t, "invalid file", filePath) - } - - return nil - }) - require.NoError(t, err, "happy path") +func TestDir_Walk(t *testing.T) { + type fields struct { + skipFiles []string + skipDirs []string + } + tests := []struct { + name string + fields fields + rootDir string + analyzeFn walker.WalkFunc + wantErr string + }{ + { + name: "happy path", + rootDir: "testdata/fs", + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + if filePath == "testdata/fs/bar" { + got, err := opener() + require.NoError(t, err) + assert.Equal(t, "bar", string(got)) + } + return nil + }, + }, + { + name: "skip file", + rootDir: "testdata/fs", + fields: fields{ + skipFiles: []string{"testdata/fs/bar"}, + }, + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + if filePath == "testdata/fs/bar" { + assert.Fail(t, "skip files error", "%s should be skipped", filePath) + } + return nil + }, + }, + { + name: "skip dir", + rootDir: "testdata/fs/", + fields: fields{ + skipDirs: []string{"/testdata/fs/app/"}, + }, + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + if strings.HasPrefix(filePath, "testdata/fs/app") { + assert.Fail(t, "skip dirs error", "%s should be skipped", filePath) + } + return nil + }, + }, + { + name: "sad path", + rootDir: "testdata/fs", + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + return errors.New("error") + }, + wantErr: "failed to analyze file", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := walker.NewDir(tt.fields.skipFiles, tt.fields.skipDirs) - // sad path - err = walker.WalkDir("testdata/fs", func(filePath string, info os.FileInfo, opener analyzer.Opener) error { - return errors.New("error") - }) - require.NotNil(t, err) - require.Contains(t, err.Error(), "failed to analyze file: error", "sad path") + err := w.Walk(tt.rootDir, tt.analyzeFn) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + assert.NoError(t, err) + }) + } } diff --git a/walker/tar.go b/walker/tar.go index 9b04017ad..f06b73147 100644 --- a/walker/tar.go +++ b/walker/tar.go @@ -3,10 +3,8 @@ package walker import ( "archive/tar" "io" - "io/ioutil" "path/filepath" "strings" - "sync" "golang.org/x/xerrors" ) @@ -16,15 +14,24 @@ const ( wh string = ".wh." ) -func WalkLayerTar(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, error) { - var opqDirs, whFiles []string +type LayerTar struct { + walker +} + +func NewLayerTar(skipFiles, skipDirs []string) LayerTar { + return LayerTar{ + walker: newWalker(skipFiles, skipDirs), + } +} + +func (w LayerTar) Walk(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, error) { + var opqDirs, whFiles, skipDirs []string tr := tar.NewReader(layer) for { hdr, err := tr.Next() if err == io.EOF { break - } - if err != nil { + } else if err != nil { return nil, nil, xerrors.Errorf("failed to extract the archive: %w", err) } @@ -45,33 +52,42 @@ func WalkLayerTar(layer io.Reader, analyzeFn WalkFunc) ([]string, []string, erro continue } - if isIgnored(filePath) { + switch hdr.Typeflag { + case tar.TypeDir: + if w.shouldSkipDir(filePath) { + skipDirs = append(skipDirs, filePath) + continue + } + case tar.TypeSymlink, tar.TypeLink, tar.TypeReg: + if w.shouldSkipFile(filePath) { + continue + } + default: + continue + } + + if underSkippedDir(filePath, skipDirs) { continue } - if hdr.Typeflag == tar.TypeSymlink || hdr.Typeflag == tar.TypeLink || hdr.Typeflag == tar.TypeReg { - err = analyzeFn(filePath, hdr.FileInfo(), tarOnceOpener(tr)) - if err != nil { - return nil, nil, xerrors.Errorf("failed to analyze file: %w", err) - } + // A symbolic/hard link or regular file will reach here. + err = analyzeFn(filePath, hdr.FileInfo(), w.fileOnceOpener(tr)) + if err != nil { + return nil, nil, xerrors.Errorf("failed to analyze file: %w", err) } } return opqDirs, whFiles, nil } -// tarOnceOpener reads a file once and the content is shared so that some analyzers can use the same data -func tarOnceOpener(r io.Reader) func() ([]byte, error) { - var once sync.Once - var b []byte - var err error - - return func() ([]byte, error) { - once.Do(func() { - b, err = ioutil.ReadAll(r) - }) +func underSkippedDir(filePath string, skipDirs []string) bool { + for _, skipDir := range skipDirs { + rel, err := filepath.Rel(skipDir, filePath) if err != nil { - return nil, xerrors.Errorf("unable to read tar file: %w", err) + return false + } + if !strings.HasPrefix(rel, "../") { + return true } - return b, nil } + return false } diff --git a/walker/tar_test.go b/walker/tar_test.go index ea63812a3..879a96788 100644 --- a/walker/tar_test.go +++ b/walker/tar_test.go @@ -3,43 +3,95 @@ package walker_test import ( "errors" "os" + "strings" "testing" - "github.com/stretchr/testify/assert" - "github.com/aquasecurity/fanal/analyzer" + "github.com/stretchr/testify/assert" + "github.com/aquasecurity/fanal/walker" "github.com/stretchr/testify/require" ) -func TestWalkLayerTar(t *testing.T) { - // happy path - f, err := os.Open("testdata/test.tar") - require.NoError(t, err) - - opqDirs, whFiles, err := walker.WalkLayerTar(f, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { - if filePath == "baz" { - b, err := opener() +func TestLayerTar_Walk(t *testing.T) { + type fields struct { + skipFiles []string + skipDirs []string + } + tests := []struct { + name string + fields fields + inputFile string + analyzeFn walker.WalkFunc + wantOpqDirs []string + wantWhFiles []string + wantErr string + }{ + { + name: "happy path", + inputFile: "testdata/test.tar", + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + return nil + }, + wantOpqDirs: []string{"etc/"}, + wantWhFiles: []string{"foo/foo"}, + }, + { + name: "skip file", + inputFile: "testdata/test.tar", + fields: fields{ + skipFiles: []string{"/app/myweb/index.html"}, + }, + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + if filePath == "app/myweb/index.html" { + assert.Fail(t, "skip files error", "%s should be skipped", filePath) + } + return nil + }, + wantOpqDirs: []string{"etc/"}, + wantWhFiles: []string{"foo/foo"}, + }, + { + name: "skip dir", + inputFile: "testdata/test.tar", + fields: fields{ + skipDirs: []string{"/app/"}, + }, + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + if strings.HasPrefix(filePath, "app") { + assert.Fail(t, "skip dirs error", "%s should be skipped", filePath) + } + return nil + }, + wantOpqDirs: []string{"etc/"}, + wantWhFiles: []string{"foo/foo"}, + }, + { + name: "sad path", + inputFile: "testdata/test.tar", + analyzeFn: func(filePath string, info os.FileInfo, opener analyzer.Opener) error { + return errors.New("error") + }, + wantErr: "failed to analyze file", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + f, err := os.Open("testdata/test.tar") require.NoError(t, err) - assert.Equal(t, "baz\n", string(b)) - } else { - require.Fail(t, "invalid file", filePath) - } - return nil - }) - assert.Equal(t, []string{"etc/"}, opqDirs) - assert.Equal(t, []string{"foo/foo"}, whFiles) - require.NoError(t, err) - require.NoError(t, f.Close()) - // sad path - f, err = os.Open("testdata/test.tar") - require.NoError(t, err) + w := walker.NewLayerTar(tt.fields.skipFiles, tt.fields.skipDirs) - _, _, err = walker.WalkLayerTar(f, func(filePath string, info os.FileInfo, opener analyzer.Opener) error { - return errors.New("error") - }) - require.EqualError(t, err, "failed to analyze file: error") - require.NoError(t, f.Close()) + gotOpqDirs, gotWhFiles, err := w.Walk(f, tt.analyzeFn) + if tt.wantErr != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.wantErr) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.wantOpqDirs, gotOpqDirs) + assert.Equal(t, tt.wantWhFiles, gotWhFiles) + }) + } } diff --git a/walker/testdata/fs/app/myweb/test.txt b/walker/testdata/fs/app/myweb/test.txt new file mode 100644 index 000000000..e69de29bb diff --git a/walker/testdata/test.tar b/walker/testdata/test.tar index 82c3c86f6..72d5e6fce 100644 Binary files a/walker/testdata/test.tar and b/walker/testdata/test.tar differ diff --git a/walker/walk.go b/walker/walk.go index b3300c2be..b340e7f7e 100644 --- a/walker/walk.go +++ b/walker/walk.go @@ -1,35 +1,93 @@ package walker import ( + "io" "os" + "path/filepath" "strings" + "sync" - "github.com/aquasecurity/fanal/analyzer" + "golang.org/x/xerrors" + "github.com/aquasecurity/fanal/analyzer" "github.com/aquasecurity/fanal/utils" ) var ( - ignoreDirs = []string{".git", "vendor"} - ignoreSystemDirs = []string{"proc", "sys"} + appDirs = []string{".git", "vendor"} + systemDirs = []string{"proc", "sys", "dev"} ) type WalkFunc func(filePath string, info os.FileInfo, opener analyzer.Opener) error -func isIgnored(filePath string) bool { +type walker struct { + skipFiles []string + skipDirs []string +} + +func newWalker(skipFiles, skipDirs []string) walker { + var cleanSkipFiles, cleanSkipDirs []string + for _, skipFile := range skipFiles { + skipFile = filepath.Clean(filepath.ToSlash(skipFile)) + skipFile = strings.TrimLeft(skipFile, "/") + cleanSkipFiles = append(cleanSkipFiles, skipFile) + } + + for _, skipDir := range append(skipDirs, systemDirs...) { + skipDir = filepath.Clean(filepath.ToSlash(skipDir)) + skipDir = strings.TrimLeft(skipDir, "/") + cleanSkipDirs = append(cleanSkipDirs, skipDir) + } + + return walker{ + skipFiles: cleanSkipFiles, + skipDirs: cleanSkipDirs, + } +} + +func (w *walker) shouldSkipFile(filePath string) bool { + filePath = filepath.ToSlash(filePath) filePath = strings.TrimLeft(filePath, "/") - for _, path := range strings.Split(filePath, utils.PathSeparator) { - if utils.StringInSlice(path, ignoreDirs) { - return true - } + + // skip files + if utils.StringInSlice(filePath, w.skipFiles) { + return true } - // skip system directories such as /sys and /proc - for _, ignore := range ignoreSystemDirs { - if strings.HasPrefix(filePath, ignore) { - return true - } + return false +} + +func (w *walker) shouldSkipDir(dir string) bool { + dir = filepath.ToSlash(dir) + dir = strings.TrimLeft(dir, "/") + + // Skip application dirs (relative path) + base := filepath.Base(dir) + if utils.StringInSlice(base, appDirs) { + return true + } + + // Skip system dirs and specified dirs (absolute path) + if utils.StringInSlice(dir, w.skipDirs) { + return true } return false } + +// fileOnceOpener opens a file once and the content is shared so that some analyzers can use the same data +func (w *walker) fileOnceOpener(r io.Reader) func() ([]byte, error) { + var once sync.Once + var b []byte + var err error + + return func() ([]byte, error) { + once.Do(func() { + b, err = io.ReadAll(r) + }) + if err != nil { + return nil, xerrors.Errorf("unable to read the file: %w", err) + } + return b, nil + } +} diff --git a/walker/walk_test.go b/walker/walk_test.go deleted file mode 100644 index 3c96d4b1a..000000000 --- a/walker/walk_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package walker - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func Test_isIgnore(t *testing.T) { - for _, fp := range ignoreDirs { - assert.True(t, isIgnored(fp)) - } - - for _, fp := range ignoreSystemDirs { - assert.True(t, isIgnored(fp)) - } - - for _, fp := range []string{"foo", "foo/bar"} { - assert.False(t, isIgnored(fp)) - } -}