From 52b8d31bc697d9cbe807a0c8205c99668701a794 Mon Sep 17 00:00:00 2001 From: Evan Gibler <20933572+egibs@users.noreply.github.com> Date: Fri, 16 Aug 2024 13:24:42 -0500 Subject: [PATCH] Use concurrency for path scanning (#405) * Use file descriptors Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Relocate file descriptor code; implement working concurrency for discovered paths Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Move to ordered map; relocate File rendering Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Limit concurrency to runtime.NumCPU() Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Rename map variables Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Directly initialize diff maps Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Appease the linter Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Configurable concurrency Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Update samples_test.go Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * More consolidation Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Add exists check Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Use -j instead of -c for concurrency Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Use an errgroup instead of a waitgroup + semaphore Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Use SetLimit; add concurrency config to all tests; linting fixes Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Move sorting code into scan.go; move rendering inside of map construction loop Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Use 8-core runner for tests Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Go back to ubuntu-latest Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Run core tests in parallel Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Better alias naming, consolidate structs Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Run make fix Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Replace mutexes with sync.Map Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Remove struct/type Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> * Appease the linter Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> --------- Signed-off-by: egibs <20933572+egibs@users.noreply.github.com> Signed-off-by: Evan Gibler <20933572+egibs@users.noreply.github.com> --- bincapz.go | 3 + go.mod | 8 +- go.sum | 19 +++-- pkg/action/archive_test.go | 23 ++---- pkg/action/diff.go | 49 +++++++----- pkg/action/oci_test.go | 36 ++++----- pkg/action/programkind.go | 2 +- pkg/action/scan.go | 133 ++++++++++++++++++++----------- pkg/action/testdata/scan_archive | 3 +- pkg/bincapz/bincapz.go | 14 ++-- pkg/render/markdown.go | 33 ++++---- pkg/render/simple.go | 25 +++--- pkg/render/stats.go | 20 +++-- pkg/render/terminal.go | 35 ++++---- 14 files changed, 219 insertions(+), 184 deletions(-) diff --git a/bincapz.go b/bincapz.go index b40b51022..84ac4ebf8 100644 --- a/bincapz.go +++ b/bincapz.go @@ -11,6 +11,7 @@ import ( "io/fs" "log/slog" "os" + "runtime" "strings" "github.com/chainguard-dev/bincapz/pkg/action" @@ -56,6 +57,7 @@ func parseRisk(s string) int { func main() { allFlag := flag.Bool("all", false, "Ignore nothing, show all") + concurrencyFlag := flag.Int("j", runtime.NumCPU(), "Concurrently scan files within target directories") diffFlag := flag.Bool("diff", false, "Show capability drift between two files") formatFlag := flag.String("format", "terminal", "Output type -- valid values are: json, markdown, simple, terminal, yaml") ignoreSelfFlag := flag.Bool("ignore-self", true, "Ignore the bincapz binary") @@ -192,6 +194,7 @@ func main() { Stats: stats, ErrFirstHit: *errFirstHitFlag, ErrFirstMiss: *errFirstMissFlag, + Concurrency: *concurrencyFlag, } var res *bincapz.Report diff --git a/go.mod b/go.mod index 20ae90fd4..c4d7320da 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/chainguard-dev/bincapz -go 1.23 +go 1.23.0 require ( github.com/agext/levenshtein v1.2.3 @@ -12,17 +12,22 @@ require ( github.com/liamg/magic v0.0.1 github.com/olekukonko/tablewriter v0.0.5 github.com/ulikunitz/xz v0.5.12 + github.com/wk8/go-ordered-map/v2 v2.1.8 + golang.org/x/sync v0.8.0 golang.org/x/term v0.23.0 gopkg.in/yaml.v3 v3.0.1 ) require ( + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect github.com/docker/cli v27.1.2+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect github.com/docker/docker-credential-helpers v0.8.2 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.2.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect @@ -33,6 +38,5 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/vbatts/tar-split v0.11.5 // indirect - golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.24.0 // indirect ) diff --git a/go.sum b/go.sum index 88227a7e7..91a30d550 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,9 @@ github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= -github.com/chainguard-dev/clog v1.4.0 h1:x0YyEppnUX+dxQAnfGNYdQEKNDSRCAwC08f/1eIxJ9E= -github.com/chainguard-dev/clog v1.4.0/go.mod h1:cV516KZWqYc/phZsCNwF36u/KMGS+Gj5Uqeb8Hlp95Y= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/chainguard-dev/clog v1.5.0 h1:VFwdxf+4x7+EG8lRO4/tZFP7Hn/NG8OVkVNfgnnsADw= github.com/chainguard-dev/clog v1.5.0/go.mod h1:4+WFhRMsGH79etYXY3plYdp+tCz/KCkU8fAr0HoaPvs= github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU= @@ -9,8 +11,6 @@ github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9N github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/docker/cli v27.1.1+incompatible h1:goaZxOqs4QKxznZjjBWKONQci/MywhtRv2oNn0GkeZE= -github.com/docker/cli v27.1.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v27.1.2+incompatible h1:nYviRv5Y+YAKx3dFrTvS1ErkyVVunKOhoweCTE1BsnI= github.com/docker/cli v27.1.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= @@ -25,6 +25,7 @@ github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8= github.com/hillu/go-yara/v4 v4.3.3 h1:O+7iYTZK20fzsXiJyvA0d529RTdnZCrgS6HdE0O7BMg= github.com/hillu/go-yara/v4 v4.3.3/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -34,14 +35,14 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/liamg/magic v0.0.1 h1:Ru22ElY+sCh6RvRTWjQzKKCxsEco8hE0co8n1qe7TBM= github.com/liamg/magic v0.0.1/go.mod h1:yQkOmZZI52EA+SQ2xyHpVw8fNvTBruF873Y+Vt6S+fk= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= -github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -69,15 +70,13 @@ github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vbatts/tar-split v0.11.5 h1:3bHCTIheBm1qFTcgh9oPu+nNBtX+XJIupG/vacinCts= github.com/vbatts/tar-split v0.11.5/go.mod h1:yZbwRsSeGjusneWgA781EKej9HF8vme8okylkAeNKLk= -golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= -golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= -golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.23.0 h1:F6D4vR+EHoL9/sWAWgAR1H2DcHr4PareCbAaCo1RpuU= diff --git a/pkg/action/archive_test.go b/pkg/action/archive_test.go index fe91a773d..2ef6162ee 100644 --- a/pkg/action/archive_test.go +++ b/pkg/action/archive_test.go @@ -6,8 +6,7 @@ import ( "io/fs" "os" "path/filepath" - "sort" - "strings" + "runtime" "testing" "github.com/chainguard-dev/bincapz/pkg/bincapz" @@ -230,11 +229,12 @@ func TestScanArchive(t *testing.T) { t.Fatalf("render: %v", err) } bc := bincapz.Config{ - IgnoreSelf: false, - IgnoreTags: []string{"harmless"}, - Renderer: simple, - Rules: yrs, - ScanPaths: []string{"testdata/apko_nested.tar.gz"}, + IgnoreSelf: false, + IgnoreTags: []string{"harmless"}, + Renderer: simple, + Rules: yrs, + ScanPaths: []string{"testdata/apko_nested.tar.gz"}, + Concurrency: runtime.NumCPU(), } res, err := Scan(ctx, bc) if err != nil { @@ -246,14 +246,7 @@ func TestScanArchive(t *testing.T) { outBytes := out.Bytes() - // Sort the output to ensure consistent ordering - sorted := func(input []byte) []byte { - lines := strings.Split(string(input), "\n") - sort.Strings(lines) - return []byte(strings.Join(lines, "\n")) - } - sortedBytes := sorted(outBytes) - got := string(sortedBytes) + got := string(outBytes) td, err := os.ReadFile("testdata/scan_archive") if err != nil { diff --git a/pkg/action/diff.go b/pkg/action/diff.go index d5cbe3e18..f72ee29f3 100644 --- a/pkg/action/diff.go +++ b/pkg/action/diff.go @@ -14,6 +14,7 @@ import ( "github.com/agext/levenshtein" "github.com/chainguard-dev/bincapz/pkg/bincapz" "github.com/chainguard-dev/clog" + orderedmap "github.com/wk8/go-ordered-map/v2" ) func relFileReport(ctx context.Context, c bincapz.Config, fromPath string) (map[string]*bincapz.FileReport, error) { @@ -25,15 +26,15 @@ func relFileReport(ctx context.Context, c bincapz.Config, fromPath string) (map[ return nil, err } fromRelPath := map[string]*bincapz.FileReport{} - for _, f := range fromReport.Files { - if f.Skipped != "" || f.Error != "" { + for files := fromReport.Files.Oldest(); files != nil; files = files.Next() { + if files.Value.Skipped != "" || files.Value.Error != "" { continue } - rel, err := filepath.Rel(fromPath, f.Path) + rel, err := filepath.Rel(fromPath, files.Value.Path) if err != nil { - return nil, fmt.Errorf("rel(%q,%q): %w", fromPath, f.Path, err) + return nil, fmt.Errorf("rel(%q,%q): %w", fromPath, files.Value.Path, err) } - fromRelPath[rel] = f + fromRelPath[rel] = files.Value } return fromRelPath, nil @@ -55,9 +56,9 @@ func Diff(ctx context.Context, c bincapz.Config) (*bincapz.Report, error) { } d := &bincapz.DiffReport{ - Added: map[string]*bincapz.FileReport{}, - Removed: map[string]*bincapz.FileReport{}, - Modified: map[string]*bincapz.FileReport{}, + Added: orderedmap.New[string, *bincapz.FileReport](), + Removed: orderedmap.New[string, *bincapz.FileReport](), + Modified: orderedmap.New[string, *bincapz.FileReport](), } processSrc(ctx, c, src, dest, d) @@ -74,7 +75,7 @@ func processSrc(ctx context.Context, c bincapz.Config, src, dest map[string]*bin for relPath, fr := range src { tr, exists := dest[relPath] if !exists { - d.Removed[relPath] = fr + d.Removed.Set(relPath, fr) continue } handleFile(ctx, c, fr, tr, relPath, d) @@ -98,7 +99,7 @@ func handleFile(ctx context.Context, c bincapz.Config, fr, tr *bincapz.FileRepor } } - d.Modified[relPath] = rbs + d.Modified.Set(relPath, rbs) } func createFileReport(tr, fr *bincapz.FileReport) *bincapz.FileReport { @@ -127,7 +128,7 @@ func processDest(ctx context.Context, c bincapz.Config, from, to map[string]*bin for relPath, tr := range to { fr, exists := from[relPath] if !exists { - d.Added[relPath] = tr + d.Added.Set(relPath, tr) continue } @@ -153,10 +154,13 @@ func fileDestination(ctx context.Context, c bincapz.Config, fr, tr *bincapz.File } // are there already modified behaviors for this file? - if _, exists := d.Modified[relPath]; !exists { - d.Modified[relPath] = abs + if _, exists := d.Modified.Get(relPath); !exists { + d.Modified.Set(relPath, abs) } else { - d.Modified[relPath].Behaviors = append(d.Modified[relPath].Behaviors, abs.Behaviors...) + if rel, exists := d.Modified.Get(relPath); exists { + rel.Behaviors = append(rel.Behaviors, abs.Behaviors...) + d.Modified.Set(relPath, rel) + } } } @@ -172,20 +176,21 @@ func combineReports(d *bincapz.DiffReport) []diffReports { diffs := make(chan diffReports) var wg sync.WaitGroup - for rpath, rfr := range d.Removed { + for removed := d.Removed.Oldest(); removed != nil; removed = removed.Next() { wg.Add(1) go func(path string, fr *bincapz.FileReport) { defer wg.Done() - for apath, afr := range d.Added { + for added := d.Added.Oldest(); added != nil; added = added.Next() { diffs <- diffReports{ - Added: apath, - AddedFR: afr, + Added: added.Key, + AddedFR: added.Value, Removed: path, RemovedFR: fr, } } - }(rpath, rfr) + }(removed.Key, removed.Value) } + go func() { wg.Wait() close(diffs) @@ -246,8 +251,8 @@ func fileMove(ctx context.Context, c bincapz.Config, fr, tr *bincapz.FileReport, // Move these into the modified list if the files are not completely different (something like ~0.3) if score > 0.3 { - d.Modified[apath] = abs - delete(d.Removed, rpath) - delete(d.Added, apath) + d.Modified.Set(apath, abs) + d.Modified.Delete(rpath) + d.Modified.Delete(apath) } } diff --git a/pkg/action/oci_test.go b/pkg/action/oci_test.go index 74c06b7e4..d6771e375 100644 --- a/pkg/action/oci_test.go +++ b/pkg/action/oci_test.go @@ -4,7 +4,7 @@ import ( "bytes" "io/fs" "os" - "regexp" + "runtime" "testing" "github.com/chainguard-dev/bincapz/pkg/bincapz" @@ -14,21 +14,13 @@ import ( thirdparty "github.com/chainguard-dev/bincapz/third_party" "github.com/chainguard-dev/clog" "github.com/chainguard-dev/clog/slogtest" + "github.com/google/go-cmp/cmp" ) -func reduceMarkdown(s string) string { - spaceRe := regexp.MustCompile(` +`) - dashRe := regexp.MustCompile(` -`) - - s = spaceRe.ReplaceAllString(s, " ") - s = dashRe.ReplaceAllString(s, " ") - return s -} - func TestOCI(t *testing.T) { t.Parallel() ctx := slogtest.Context(t) - clog.FromContext(ctx).With("test", "scan_archive") + clog.FromContext(ctx).With("test", "scan_oci") yrs, err := compile.Recursive(ctx, []fs.FS{rules.FS, thirdparty.FS}) if err != nil { @@ -42,12 +34,13 @@ func TestOCI(t *testing.T) { } bc := bincapz.Config{ - IgnoreSelf: false, - IgnoreTags: []string{"harmless"}, - Renderer: simple, - Rules: yrs, - ScanPaths: []string{"cgr.dev/chainguard/static"}, - OCI: true, + IgnoreSelf: false, + IgnoreTags: []string{"harmless"}, + Renderer: simple, + Rules: yrs, + ScanPaths: []string{"cgr.dev/chainguard/static"}, + OCI: true, + Concurrency: runtime.NumCPU(), } res, err := Scan(ctx, bc) if err != nil { @@ -57,14 +50,15 @@ func TestOCI(t *testing.T) { t.Fatalf("full: %v", err) } - got := reduceMarkdown(out.String()) + got := out.String() td, err := os.ReadFile("testdata/scan_oci") if err != nil { t.Fatalf("testdata read failed: %v", err) } - want := reduceMarkdown(string(td)) - if got != want { - t.Fatalf("got %q, want %q", got, want) + // Sort the loaded contents to ensure consistent ordering + want := string(td) + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("Simple output mismatch: (-want +got):\n%s", diff) } } diff --git a/pkg/action/programkind.go b/pkg/action/programkind.go index b52f33d03..b02dc0883 100644 --- a/pkg/action/programkind.go +++ b/pkg/action/programkind.go @@ -80,7 +80,7 @@ func programKind(ctx context.Context, path string) string { headerString := "" n, err := io.ReadFull(f, header[:]) if err == nil || errors.Is(err, io.ErrUnexpectedEOF) { - kind, err := magic.Lookup(header[:n]) + kind, err := magic.LookupSync(header[:n]) if err == nil { desc = kind.Description } diff --git a/pkg/action/scan.go b/pkg/action/scan.go index 07a66595f..27e0f7541 100644 --- a/pkg/action/scan.go +++ b/pkg/action/scan.go @@ -9,14 +9,18 @@ import ( "log/slog" "os" "path/filepath" + "slices" "sort" "strings" + "sync" "github.com/chainguard-dev/bincapz/pkg/bincapz" "github.com/chainguard-dev/bincapz/pkg/render" "github.com/chainguard-dev/bincapz/pkg/report" "github.com/chainguard-dev/clog" "github.com/hillu/go-yara/v4" + orderedmap "github.com/wk8/go-ordered-map/v2" + "golang.org/x/sync/errgroup" ) // findFilesRecurslively returns a list of files found recursively within a path. @@ -158,7 +162,7 @@ func recursiveScan(ctx context.Context, c bincapz.Config) (*bincapz.Report, erro logger := clog.FromContext(ctx) logger.Debug("recursive scan", slog.Any("config", c)) r := &bincapz.Report{ - Files: map[string]*bincapz.FileReport{}, + Files: orderedmap.New[string, *bincapz.FileReport](), } if len(c.IgnoreTags) > 0 { r.Filter = strings.Join(c.IgnoreTags, ",") @@ -173,6 +177,7 @@ func recursiveScan(ctx context.Context, c bincapz.Config) (*bincapz.Report, erro scanPathFindings := map[string]*bincapz.FileReport{} + var results sync.Map for _, scanPath := range c.ScanPaths { logger.Debug("recursive scan", slog.Any("scanPath", scanPath)) imageURI := "" @@ -196,8 +201,20 @@ func recursiveScan(ctx context.Context, c bincapz.Config) (*bincapz.Report, erro } logger.Debug("files found", slog.Any("path count", len(paths)), slog.Any("scanPath", scanPath)) + maxConcurrency := c.Concurrency + if maxConcurrency < 1 { + maxConcurrency = 1 + } + // path refers to a real local path, not the requested scanPath + pc := make(chan string, len(paths)) for _, path := range paths { + pc <- path + } + close(pc) + + process := func(path string) error { + //nolint:nestif // ignore complexity of 13 if isSupportedArchive(path) { logger.Debug("found archive path", slog.Any("path", path)) frs, err := processArchive(ctx, c, yrs, path, logger) @@ -208,39 +225,68 @@ func recursiveScan(ctx context.Context, c bincapz.Config) (*bincapz.Report, erro // If we're handling an archive within an OCI archive, wait to for other files to declare a miss if !c.OCI { if err := errIfHitOrMiss(frs, "archive", path, c.ErrFirstHit, c.ErrFirstMiss); err != nil { - logger.Debugf("match short circuit: %v", err) - return r, err + results.Store(path, &bincapz.FileReport{}) + return err } } for extractedPath, fr := range frs { - scanPathFindings[extractedPath] = fr + results.Store(extractedPath, fr) + } + } else { + trimPath := "" + if c.OCI { + scanPath = imageURI + trimPath = ociExtractPath } - continue - } - trimPath := "" - if c.OCI { - scanPath = imageURI - trimPath = ociExtractPath + logger.Debug("processing path", slog.Any("path", path)) + fr, err := processFile(ctx, c, yrs, path, scanPath, trimPath, logger) + if err != nil { + results.Store(path, &bincapz.FileReport{}) + return err + } + if fr != nil { + results.Store(path, fr) + if !c.OCI { + if err := errIfHitOrMiss(map[string]*bincapz.FileReport{path: fr}, "file", path, c.ErrFirstHit, c.ErrFirstMiss); err != nil { + logger.Debugf("match short circuit: %s", err) + results.Store(path, &bincapz.FileReport{}) + } + } + } } + return nil + } - logger.Debug("processing path", slog.Any("path", path)) - fr, err := processFile(ctx, c, yrs, path, scanPath, trimPath, logger) - if err != nil { - return r, err - } - if fr == nil { - continue - } - scanPathFindings[path] = fr - if !c.OCI { - if err := errIfHitOrMiss(map[string]*bincapz.FileReport{path: fr}, "file", path, c.ErrFirstHit, c.ErrFirstMiss); err != nil { - logger.Debugf("match short circuit: %s", err) - return r, err + var g errgroup.Group + g.SetLimit(maxConcurrency) + for path := range pc { + path := path + g.Go(func() error { + return process(path) + }) + } + + if err := g.Wait(); err != nil { + logger.Errorf("error with processing %v\n", err) + } + + var pathKeys []string + results.Range(func(key, value interface{}) bool { + if k, ok := key.(string); ok { + func(key string) { + pathKeys = append(pathKeys, key) + slices.Sort(pathKeys) + }(k) + value, ok := value.(*bincapz.FileReport) + if !ok { + return false } + scanPathFindings[k] = value } - } + return true + }) // OCI images hadle their match his/miss logic per scanPath if c.OCI { @@ -253,12 +299,21 @@ func recursiveScan(ctx context.Context, c bincapz.Config) (*bincapz.Report, erro } } - for path, fr := range scanPathFindings { - r.Files[path] = fr + // Add the sorted paths and file reports to the parent report and render the results + for _, k := range pathKeys { + r.Files.Set(k, scanPathFindings[k]) + if c.Renderer != nil && r.Diff == nil { + if scanPathFindings[k].RiskScore < c.MinFileRisk { + return nil, nil + } + + if err := c.Renderer.File(ctx, scanPathFindings[k]); err != nil { + return nil, fmt.Errorf("render: %w", err) + } + } } } // loop: next scan path - - logger.Debugf("recursive scan complete: %d files", len(r.Files)) + logger.Debugf("recursive scan complete: %d files", r.Files.Len()) return r, nil } @@ -302,7 +357,7 @@ func processFile(ctx context.Context, c bincapz.Config, yrs *yara.Rules, path st fr, err := scanSinglePath(ctx, c, yrs, path, scanPath, archiveRoot) if err != nil { logger.Errorf("scan path: %v", err) - return nil, nil + return nil, err } if fr == nil { @@ -312,18 +367,7 @@ func processFile(ctx context.Context, c bincapz.Config, yrs *yara.Rules, path st if fr.Error != "" { logger.Errorf("scan error: %s", fr.Error) - return nil, nil - } - - if c.Renderer != nil { - if fr.RiskScore < c.MinFileRisk { - // logger.Infof("%s [%d] does not meet min file risk [%d]", path, fr.RiskScore, c.MinFileRisk) - return nil, nil - } - - if err := c.Renderer.File(ctx, fr); err != nil { - return nil, fmt.Errorf("render: %w", err) - } + return nil, fmt.Errorf("report error: %v", fr.Error) } return fr, nil @@ -335,12 +379,11 @@ func Scan(ctx context.Context, c bincapz.Config) (*bincapz.Report, error) { if err != nil { return r, err } - for path, rf := range r.Files { - if rf.RiskScore < c.MinFileRisk { - delete(r.Files, path) + for files := r.Files.Oldest(); files != nil; files = files.Next() { + if files.Value.RiskScore < c.MinFileRisk { + r.Files.Delete(files.Key) } } - if c.Stats { err = render.Statistics(r) if err != nil { diff --git a/pkg/action/testdata/scan_archive b/pkg/action/testdata/scan_archive index 21046a742..c480cc44a 100644 --- a/pkg/action/testdata/scan_archive +++ b/pkg/action/testdata/scan_archive @@ -1,4 +1,3 @@ - # testdata/apko_nested.tar.gz ∴ /apko_0.13.2_linux_arm64/apko archives/zip combo/dropper/shell @@ -116,4 +115,4 @@ secrets/ssh security_controls/linux/selinux shell/background/sleep shell/exec -time/clock/set \ No newline at end of file +time/clock/set diff --git a/pkg/bincapz/bincapz.go b/pkg/bincapz/bincapz.go index b99a85397..e387e6a20 100644 --- a/pkg/bincapz/bincapz.go +++ b/pkg/bincapz/bincapz.go @@ -8,6 +8,7 @@ import ( "io" "github.com/hillu/go-yara/v4" + orderedmap "github.com/wk8/go-ordered-map/v2" ) // Renderer is a common interface for Renderers. @@ -31,6 +32,7 @@ type Config struct { Stats bool ErrFirstMiss bool ErrFirstHit bool + Concurrency int } type Behavior struct { @@ -84,15 +86,15 @@ type FileReport struct { } type DiffReport struct { - Added map[string]*FileReport `json:",omitempty" yaml:",omitempty"` - Removed map[string]*FileReport `json:",omitempty" yaml:",omitempty"` - Modified map[string]*FileReport `json:",omitempty" yaml:",omitempty"` + Added *orderedmap.OrderedMap[string, *FileReport] `json:",omitempty" yaml:",omitempty"` + Removed *orderedmap.OrderedMap[string, *FileReport] `json:",omitempty" yaml:",omitempty"` + Modified *orderedmap.OrderedMap[string, *FileReport] `json:",omitempty" yaml:",omitempty"` } type Report struct { - Files map[string]*FileReport `json:",omitempty" yaml:",omitempty"` - Diff *DiffReport `json:",omitempty" yaml:",omitempty"` - Filter string `json:",omitempty" yaml:",omitempty"` + Files *orderedmap.OrderedMap[string, *FileReport] `json:",omitempty" yaml:",omitempty"` + Diff *DiffReport `json:",omitempty" yaml:",omitempty"` + Filter string `json:",omitempty" yaml:",omitempty"` } type IntMetric struct { diff --git a/pkg/render/markdown.go b/pkg/render/markdown.go index 97e08e7b3..634f0f9fd 100644 --- a/pkg/render/markdown.go +++ b/pkg/render/markdown.go @@ -42,7 +42,7 @@ func matchFragmentLink(s string) string { } func (r Markdown) File(ctx context.Context, fr *bincapz.FileReport) error { - if len(fr.Behaviors) != 0 { + if len(fr.Behaviors) > 0 { markdownTable(ctx, fr, r.w, tableConfig{Title: fmt.Sprintf("## %s [%s]", fr.Path, mdRisk(fr.RiskScore, fr.RiskLevel))}) } return nil @@ -53,35 +53,32 @@ func (r Markdown) Full(ctx context.Context, rep *bincapz.Report) error { return nil } - for f, fr := range rep.Diff.Removed { - fr := fr - markdownTable(ctx, fr, r.w, tableConfig{Title: fmt.Sprintf("## Deleted: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffRemoved: true}) + for removed := rep.Diff.Removed.Oldest(); removed != nil; removed = removed.Next() { + markdownTable(ctx, removed.Value, r.w, tableConfig{Title: fmt.Sprintf("## Deleted: %s [%s]", removed.Key, mdRisk(removed.Value.RiskScore, removed.Value.RiskLevel)), DiffRemoved: true}) } - for f, fr := range rep.Diff.Added { - fr := fr - markdownTable(ctx, fr, r.w, tableConfig{Title: fmt.Sprintf("## Added: %s [%s]", f, mdRisk(fr.RiskScore, fr.RiskLevel)), DiffAdded: true}) + for added := rep.Diff.Added.Oldest(); added != nil; added = added.Next() { + markdownTable(ctx, added.Value, r.w, tableConfig{Title: fmt.Sprintf("## Added: %s [%s]", added.Key, mdRisk(added.Value.RiskScore, added.Value.RiskLevel)), DiffAdded: true}) } - for _, fr := range rep.Diff.Modified { - fr := fr + for modified := rep.Diff.Modified.Oldest(); modified != nil; modified = modified.Next() { var title string - if fr.PreviousRelPath != "" && fr.PreviousRelPathScore >= 0.9 { - title = fmt.Sprintf("## Moved: %s -> %s (similarity: %0.2f)", fr.PreviousRelPath, fr.Path, fr.PreviousRelPathScore) + if modified.Value.PreviousRelPath != "" && modified.Value.PreviousRelPathScore >= 0.9 { + title = fmt.Sprintf("## Moved: %s -> %s (similarity: %0.2f)", modified.Value.PreviousRelPath, modified.Value.Path, modified.Value.PreviousRelPathScore) } else { - title = fmt.Sprintf("## Changed: %s", fr.Path) + title = fmt.Sprintf("## Changed: %s", modified.Value.Path) } - if fr.RiskScore != fr.PreviousRiskScore { + if modified.Value.RiskScore != modified.Value.PreviousRiskScore { title = fmt.Sprintf("%s [%s → %s]", title, - mdRisk(fr.PreviousRiskScore, fr.PreviousRiskLevel), - mdRisk(fr.RiskScore, fr.RiskLevel)) + mdRisk(modified.Value.PreviousRiskScore, modified.Value.PreviousRiskLevel), + mdRisk(modified.Value.RiskScore, modified.Value.RiskLevel)) } fmt.Fprint(r.w, title+"\n\n") added := 0 removed := 0 - for _, b := range fr.Behaviors { + for _, b := range modified.Value.Behaviors { if b.DiffAdded { added++ } @@ -91,14 +88,14 @@ func (r Markdown) Full(ctx context.Context, rep *bincapz.Report) error { } if added > 0 { - markdownTable(ctx, fr, r.w, tableConfig{ + markdownTable(ctx, modified.Value, r.w, tableConfig{ Title: fmt.Sprintf("### %d new behaviors", added), SkipRemoved: true, }) } if removed > 0 { - markdownTable(ctx, fr, r.w, tableConfig{ + markdownTable(ctx, modified.Value, r.w, tableConfig{ Title: fmt.Sprintf("### %d removed behaviors", removed), SkipAdded: true, }) diff --git a/pkg/render/simple.go b/pkg/render/simple.go index b0b9d42c5..fc4a6388e 100644 --- a/pkg/render/simple.go +++ b/pkg/render/simple.go @@ -25,7 +25,7 @@ func (r Simple) File(_ context.Context, fr *bincapz.FileReport) error { return nil } - if len(fr.Behaviors) != 0 { + if len(fr.Behaviors) > 0 { fmt.Fprintf(r.w, "# %s\n", fr.Path) } @@ -48,11 +48,11 @@ func (r Simple) Full(_ context.Context, rep *bincapz.Report) error { return nil } - for f, fr := range rep.Diff.Removed { - fmt.Fprintf(r.w, "--- missing: %s\n", f) + for removed := rep.Diff.Removed.Oldest(); removed != nil; removed = removed.Next() { + fmt.Fprintf(r.w, "--- missing: %s\n", removed.Key) var bs []*bincapz.Behavior - bs = append(bs, fr.Behaviors...) + bs = append(bs, removed.Value.Behaviors...) sort.Slice(bs, func(i, j int) bool { return bs[i].ID < bs[j].ID @@ -63,11 +63,11 @@ func (r Simple) Full(_ context.Context, rep *bincapz.Report) error { } } - for f, fr := range rep.Diff.Added { - fmt.Fprintf(r.w, "++++ added: %s\n", f) + for added := rep.Diff.Added.Oldest(); added != nil; added = added.Next() { + fmt.Fprintf(r.w, "++++ added: %s\n", added.Key) var bs []*bincapz.Behavior - bs = append(bs, fr.Behaviors...) + bs = append(bs, added.Value.Behaviors...) sort.Slice(bs, func(i, j int) bool { return bs[i].ID < bs[j].ID @@ -78,15 +78,15 @@ func (r Simple) Full(_ context.Context, rep *bincapz.Report) error { } } - for _, fr := range rep.Diff.Modified { - if fr.PreviousRelPath != "" && fr.PreviousRelPathScore >= 0.9 { - fmt.Fprintf(r.w, ">>> moved: %s -> %s (score: %f)\n", fr.PreviousRelPath, fr.Path, fr.PreviousRelPathScore) + for modified := rep.Diff.Modified.Oldest(); modified != nil; modified = modified.Next() { + if modified.Value.PreviousRelPath != "" && modified.Value.PreviousRelPathScore >= 0.9 { + fmt.Fprintf(r.w, ">>> moved: %s -> %s (score: %f)\n", modified.Value.PreviousRelPath, modified.Value.Path, modified.Value.PreviousRelPathScore) } else { - fmt.Fprintf(r.w, "*** changed: %s\n", fr.Path) + fmt.Fprintf(r.w, "*** changed: %s\n", modified.Value.Path) } var bs []*bincapz.Behavior - bs = append(bs, fr.Behaviors...) + bs = append(bs, modified.Value.Behaviors...) sort.Slice(bs, func(i, j int) bool { return bs[i].ID < bs[j].ID @@ -103,5 +103,6 @@ func (r Simple) Full(_ context.Context, rep *bincapz.Report) error { } } } + return nil } diff --git a/pkg/render/stats.go b/pkg/render/stats.go index 4c20a0795..b2e4f100d 100644 --- a/pkg/render/stats.go +++ b/pkg/render/stats.go @@ -6,23 +6,21 @@ import ( "github.com/chainguard-dev/bincapz/pkg/bincapz" "github.com/chainguard-dev/bincapz/pkg/report" + orderedmap "github.com/wk8/go-ordered-map/v2" ) -func riskStatistics(files map[string]*bincapz.FileReport) ([]bincapz.IntMetric, int, int) { +func riskStatistics(files *orderedmap.OrderedMap[string, *bincapz.FileReport]) ([]bincapz.IntMetric, int, int) { riskMap := make(map[int][]string) riskStats := make(map[int]float64) // as opposed to skipped files processedFiles := 0 - for path, rf := range files { - if rf.Skipped != "" { + for files := files.Oldest(); files != nil; files = files.Next() { + if files.Value.Skipped != "" { continue } processedFiles++ - if rf.Skipped != "" { - continue - } - riskMap[rf.RiskScore] = append(riskMap[rf.RiskScore], path) + riskMap[files.Value.RiskScore] = append(riskMap[files.Value.RiskScore], files.Value.Path) for riskLevel := range riskMap { riskStats[riskLevel] = (float64(len(riskMap[riskLevel])) / float64(processedFiles)) * 100 } @@ -46,12 +44,12 @@ func riskStatistics(files map[string]*bincapz.FileReport) ([]bincapz.IntMetric, return stats, total(), processedFiles } -func pkgStatistics(files map[string]*bincapz.FileReport) ([]bincapz.StrMetric, int, int) { +func pkgStatistics(files *orderedmap.OrderedMap[string, *bincapz.FileReport]) ([]bincapz.StrMetric, int, int) { numNamespaces := 0 pkgMap := make(map[string]int) pkg := make(map[string]float64) - for _, rf := range files { - for _, namespace := range rf.Behaviors { + for files := files.Oldest(); files != nil; files = files.Next() { + for _, namespace := range files.Value.Behaviors { numNamespaces++ pkgMap[namespace.ID]++ } @@ -88,7 +86,7 @@ func Statistics(r *bincapz.Report) error { pkgSymbol := "📦" fmt.Printf("%s Statistics\n", statsSymbol) fmt.Println("---") - fmt.Printf("\033[1;37m%-15s \033[1;37m%s\033[0m\n", "Files Scanned", fmt.Sprintf("%d (%d skipped)", totalFilesProcessed, len(r.Files)-totalFilesProcessed)) + fmt.Printf("\033[1;37m%-15s \033[1;37m%s\033[0m\n", "Files Scanned", fmt.Sprintf("%d (%d skipped)", totalFilesProcessed, r.Files.Len()-totalFilesProcessed)) fmt.Printf("\033[1;37m%-15s \033[1;37m%s\033[0m\n", "Total Risks", fmt.Sprintf("%d", totalRisks)) fmt.Println("---") fmt.Printf("%s Risk Level Percentage\n", riskSymbol) diff --git a/pkg/render/terminal.go b/pkg/render/terminal.go index e332cd49f..4d9bbcc4e 100644 --- a/pkg/render/terminal.go +++ b/pkg/render/terminal.go @@ -91,7 +91,7 @@ func ShortRisk(s string) string { } func (r Terminal) File(ctx context.Context, fr *bincapz.FileReport) error { - if len(fr.Behaviors) != 0 { + if len(fr.Behaviors) > 0 { renderTable(ctx, fr, r.w, tableConfig{ Title: fmt.Sprintf("%s %s", fr.Path, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))), @@ -107,40 +107,37 @@ func (r Terminal) Full(ctx context.Context, rep *bincapz.Report) error { return nil } - for f, fr := range rep.Diff.Removed { - fr := fr - renderTable(ctx, fr, r.w, tableConfig{ - Title: fmt.Sprintf("Deleted: %s %s", f, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))), + for removed := rep.Diff.Removed.Oldest(); removed != nil; removed = removed.Next() { + renderTable(ctx, removed.Value, r.w, tableConfig{ + Title: fmt.Sprintf("Deleted: %s %s", removed.Key, darkBrackets(decorativeRisk(removed.Value.RiskScore, removed.Value.RiskLevel))), DiffRemoved: true, }) } - for f, fr := range rep.Diff.Added { - fr := fr - renderTable(ctx, fr, r.w, tableConfig{ - Title: fmt.Sprintf("Added: %s %s", f, darkBrackets(decorativeRisk(fr.RiskScore, fr.RiskLevel))), + for added := rep.Diff.Added.Oldest(); added != nil; added = added.Next() { + renderTable(ctx, added.Value, r.w, tableConfig{ + Title: fmt.Sprintf("Added: %s %s", added.Key, darkBrackets(decorativeRisk(added.Value.RiskScore, added.Value.RiskLevel))), DiffAdded: true, }) } - for _, fr := range rep.Diff.Modified { - fr := fr + for modified := rep.Diff.Modified.Oldest(); modified != nil; modified = modified.Next() { var title string - if fr.PreviousRelPath != "" && fr.PreviousRelPathScore >= 0.9 { - title = fmt.Sprintf("Moved: %s -> %s (score: %f)", fr.PreviousRelPath, fr.Path, fr.PreviousRelPathScore) + if modified.Value.PreviousRelPath != "" && modified.Value.PreviousRelPathScore >= 0.9 { + title = fmt.Sprintf("Moved: %s -> %s (score: %f)", modified.Value.PreviousRelPath, modified.Value.Path, modified.Value.PreviousRelPathScore) } else { - title = fmt.Sprintf("Changed: %s", fr.Path) + title = fmt.Sprintf("Changed: %s", modified.Value.Path) } - if fr.RiskScore != fr.PreviousRiskScore { + if modified.Value.RiskScore != modified.Value.PreviousRiskScore { title = fmt.Sprintf("%s %s\n\n", title, - darkBrackets(fmt.Sprintf("%s %s %s", decorativeRisk(fr.PreviousRiskScore, fr.PreviousRiskLevel), color.HiWhiteString("→"), decorativeRisk(fr.RiskScore, fr.RiskLevel)))) + darkBrackets(fmt.Sprintf("%s %s %s", decorativeRisk(modified.Value.PreviousRiskScore, modified.Value.PreviousRiskLevel), color.HiWhiteString("→"), decorativeRisk(modified.Value.RiskScore, modified.Value.RiskLevel)))) } fmt.Fprint(r.w, title) added := 0 removed := 0 - for _, b := range fr.Behaviors { + for _, b := range modified.Value.Behaviors { if b.DiffAdded { added++ } @@ -150,14 +147,14 @@ func (r Terminal) Full(ctx context.Context, rep *bincapz.Report) error { } if added > 0 { - renderTable(ctx, fr, r.w, tableConfig{ + renderTable(ctx, modified.Value, r.w, tableConfig{ Title: color.HiWhiteString("+++ ADDED: %d behavior(s) +++", added), SkipRemoved: true, }) } if removed > 0 { - renderTable(ctx, fr, r.w, tableConfig{ + renderTable(ctx, modified.Value, r.w, tableConfig{ Title: color.HiWhiteString("--- REMOVED: %d behavior(s) ---", removed), SkipAdded: true, })