Skip to content

Commit

Permalink
Swap over to yara-x; improve performance and readability
Browse files Browse the repository at this point in the history
Signed-off-by: egibs <[email protected]>
  • Loading branch information
egibs committed Dec 23, 2024
1 parent fcd1105 commit c26df49
Show file tree
Hide file tree
Showing 56 changed files with 2,349 additions and 1,939 deletions.
20 changes: 5 additions & 15 deletions .github/workflows/go-tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,16 @@ permissions:
jobs:
test:
runs-on: mal-ubuntu-latest-8-core

container: cgr.dev/chainguard/wolfi-base:latest
steps:
- uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
with:
egress-policy: audit
- name: Install dependencies
run: |
apk update
apk add curl findutils git go nodejs xz yara-x-compat
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version-file: go.mod
check-latest: true

- uses: chainguard-dev/actions/goimports@9d943fc9889a0c0795e3c2bd4b949a9b610ac02e # main

- name: install dependencies
run: |
sudo apt update && sudo apt install libyara-dev xz-utils -y
- name: Unit tests
run: |
make test
Expand Down
32 changes: 10 additions & 22 deletions .github/workflows/style.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -114,29 +114,17 @@ jobs:

golangci-lint:
name: golangci-lint
runs-on: ubuntu-24.04

runs-on: ubuntu-latest
container: cgr.dev/chainguard/wolfi-base:latest
steps:
- name: Harden Runner
uses: step-security/harden-runner@0080882f6c36860b6ba35c610c98ce87d4e2f26f # v2.10.2
with:
egress-policy: audit

- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Set up Go
uses: actions/setup-go@3041bf56c941b39c61721a86cd11f3bb1338122a # v5.2.0
with:
go-version-file: go.mod
check-latest: true

- name: install libyara-dev
- name: Install dependencies
run: |
sudo apt update && sudo apt install libyara-dev -y
apk update
apk add git go yara-x-compat
- name: golangci-lint
uses: golangci/golangci-lint-action@971e284b6050e8a5849b72094c50ab08da042db8 # v6.1.1
with:
version: v1.62.0
args: --timeout=5m
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

- name: Run golangci-lint
run: |
make golangci-lint-lint
26 changes: 24 additions & 2 deletions .github/workflows/third-party.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@ jobs:
update:
if: ${{ github.repository }} == 'chainguard-dev/malcontent'
runs-on: mal-ubuntu-latest-8-core
env:
PKG_CONFIG_PATH: ${{ github.workspace }}
LD_LIBRARY_PATH: ${{ github.workspace }}
permissions:
contents: write
id-token: write
Expand All @@ -28,9 +31,28 @@ jobs:
with:
scope: chainguard-dev/malcontent
identity: third-party
- name: Install yara and libyara-dev
- name: Clone yara-x
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: VirusTotal/yara-x
path: yara-x

- name: Setup Rust
uses: dtolnay/rust-toolchain@315e265cd78dad1e1dcf3a5074f6d6c47029d5aa # ???
with:
toolchain: stable

- name: Install yara-x-capi
run: |
sudo apt update && sudo apt install yara libyara-dev -y
cd ${{ github.workspace }}/yara-x
cargo install cargo-c
cargo cinstall -p yara-x-capi --release --destdir /tmp/yara-x
sudo mkdir -p /usr/local/lib
sudo mkdir -p /usr/local/include
sudo cp /tmp/yara-x/usr/local/include/yara_x.h /usr/local/include/
sudo cp -a /tmp/yara-x/usr/local/lib/x86_64-linux-gnu/* /usr/local/lib/
sudo ldconfig -v
cd ${{ github.workspace }}
- name: Run make update-third-party
run: |
make update-third-party
Expand Down
16 changes: 15 additions & 1 deletion cmd/mal/mal.go
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,19 @@ func main() {
rfs = append(rfs, thirdparty.FS)
}

yrs, err := action.CachedRules(ctx, rfs)
if err != nil {
returnCode = ExitInvalidRules
}

var pool *malcontent.ScannerPool
if mc.ScannerPool == nil {
pool, err = malcontent.NewScannerPool(yrs, runtime.NumCPU())
if err != nil {
returnCode = ExitInvalidRules
}
}

mc = malcontent.Config{
Concurrency: concurrencyFlag,
ExitFirstHit: exitFirstHitFlag,
Expand All @@ -251,8 +264,9 @@ func main() {
OCI: ociFlag,
QuantityIncreasesRisk: quantityIncreasesRiskFlag,
Renderer: renderer,
RuleFS: rfs,
Rules: yrs,
ScanPaths: scanPaths,
ScannerPool: pool,
Stats: statsFlag,
}

Expand Down
19 changes: 10 additions & 9 deletions go.mod
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
module github.com/chainguard-dev/malcontent

go 1.23.3
go 1.23.4

require (
github.com/VirusTotal/yara-x/go v0.12.0
github.com/agext/levenshtein v1.2.3
github.com/cavaliergopher/cpio v1.0.1
github.com/cavaliergopher/rpm v1.2.0
Expand All @@ -15,7 +16,7 @@ require (
github.com/gabriel-vasile/mimetype v1.4.7
github.com/google/go-cmp v0.6.0
github.com/google/go-containerregistry v0.20.2
github.com/hillu/go-yara/v4 v4.3.3
github.com/klauspost/compress v1.17.11
github.com/olekukonko/tablewriter v0.0.5
github.com/shirou/gopsutil/v4 v4.24.11
github.com/ulikunitz/xz v0.5.12
Expand All @@ -30,21 +31,20 @@ require (
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/charmbracelet/x/ansi v0.4.5 // indirect
github.com/charmbracelet/x/ansi v0.6.0 // indirect
github.com/charmbracelet/x/term v0.2.1 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.15.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.5 // indirect
github.com/docker/cli v27.3.1+incompatible // indirect
github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect
github.com/docker/cli v27.4.1+incompatible // indirect
github.com/docker/distribution v2.8.3+incompatible // indirect
github.com/docker/docker-credential-helpers v0.8.2 // indirect
github.com/ebitengine/purego v0.8.1 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/klauspost/compress v1.17.11 // indirect
github.com/kr/pretty v0.2.1 // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mailru/easyjson v0.9.0 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
Expand All @@ -66,8 +66,9 @@ require (
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/net v0.33.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
google.golang.org/protobuf v1.36.0 // indirect
pault.ag/go/topsort v0.1.1 // indirect
)
31 changes: 16 additions & 15 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
github.com/VirusTotal/yara-x/go v0.12.0 h1:nR7MKfNOOqsLXyeJYjgFOQcCjfME9qAoDLiN2Z7/pcg=
github.com/VirusTotal/yara-x/go v0.12.0/go.mod h1:lgXP/nkYX349MVowrtTtU5hzMdCOWQLv3+wKll9+0F8=
github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
Expand All @@ -18,19 +20,19 @@ github.com/charmbracelet/bubbletea v1.2.4 h1:KN8aCViA0eps9SCOThb2/XPIlea3ANJLUkv
github.com/charmbracelet/bubbletea v1.2.4/go.mod h1:Qr6fVQw+wX7JkWWkVyXYk/ZUQ92a6XNekLXa3rR18MM=
github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg=
github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo=
github.com/charmbracelet/x/ansi v0.4.5 h1:LqK4vwBNaXw2AyGIICa5/29Sbdq58GbGdFngSexTdRM=
github.com/charmbracelet/x/ansi v0.4.5/go.mod h1:dk73KoMTT5AX5BsX0KrqhsTqAnhZZoCBjs7dGWp4Ktw=
github.com/charmbracelet/x/ansi v0.6.0 h1:qOznutrb93gx9oMiGf7caF7bqqubh6YIM0SWKyA08pA=
github.com/charmbracelet/x/ansi v0.6.0/go.mod h1:KBUFw1la39nl0dLl10l5ORDAqGXaeurTQmwyyVKse/Q=
github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ=
github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg=
github.com/containerd/stargz-snapshotter/estargz v0.15.1 h1:eXJjw9RbkLFgioVaTG+G/ZW/0kEe2oEKCdS/ZxIyoCU=
github.com/containerd/stargz-snapshotter/estargz v0.15.1/go.mod h1:gr2RNwukQ/S9Nv33Lt6UC7xEx58C+LHRdoqbEKjz1Kk=
github.com/cpuguy83/go-md2man/v2 v2.0.5 h1:ZtcqGrnekaHpVLArFSe4HK5DoKx1T0rq2DwVB0alcyc=
github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/containerd/stargz-snapshotter/estargz v0.16.3 h1:7evrXtoh1mSbGj/pfRccTampEyKpjpOnS3CyiV1Ebr8=
github.com/containerd/stargz-snapshotter/estargz v0.16.3/go.mod h1:uyr4BfYfOj3G9WBVE8cOlQmXAbPN9VEQpBBeJIuOipU=
github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/docker/cli v27.3.1+incompatible h1:qEGdFBF3Xu6SCvCYhc7CzaQTlBmqDuzxPDpigSyeKQQ=
github.com/docker/cli v27.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/cli v27.4.1+incompatible h1:VzPiUlRJ/xh+otB75gva3r05isHMo5wXDfPRi5/b4hI=
github.com/docker/cli v27.4.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk=
github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
github.com/docker/docker-credential-helpers v0.8.2 h1:bX3YxiGzFP5sOXWc3bTPEXdEaZSeVMrFgOr3T+zrFAo=
Expand All @@ -52,9 +54,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-containerregistry v0.20.2 h1:B1wPJ1SN/S7pB+ZAimcciVD+r+yV/l/DSArMxlbwseo=
github.com/google/go-containerregistry v0.20.2/go.mod h1:z38EKdKh4h7IP2gSfUUqEvalZBqs6AoLeWfUy34nQC8=
github.com/hillu/go-yara/v4 v4.3.3 h1:O+7iYTZK20fzsXiJyvA0d529RTdnZCrgS6HdE0O7BMg=
github.com/hillu/go-yara/v4 v4.3.3/go.mod h1:AHEs/FXVMQKVVlT6iG9d+q1BRr0gq0WoAWZQaZ0gS7s=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
Expand All @@ -66,8 +65,8 @@ github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69
github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683 h1:7UMa6KCCMjZEMDtTVdcGu0B1GmmC7QJKiCCjyTAWQy0=
github.com/lufia/plan9stats v0.0.0-20240909124753-873cd0166683/go.mod h1:ilwx/Dta8jXAgpFYFvSWEMwxmbWXyiUHkd5FwyKhb5k=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4=
github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
Expand Down Expand Up @@ -129,8 +128,8 @@ github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
Expand All @@ -146,6 +145,8 @@ golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ=
google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
Expand Down
62 changes: 62 additions & 0 deletions pkg/action/path.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package action

import (
"context"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/chainguard-dev/clog"
)

// findFilesRecursively returns a list of files found recursively within a path.
func findFilesRecursively(ctx context.Context, rootPath string) ([]string, error) {
logger := clog.FromContext(ctx)
var files []string

// Follow symlink if provided at the root
root, err := filepath.EvalSymlinks(rootPath)
if err != nil {
// If the target does not exist, log the error but return gracefully
// This is useful when scanning -compat packages
if os.IsNotExist(err) {
logger.Debugf("symlink target does not exist: %s", err.Error())
return nil, nil
}
// Allow /proc/XXX/exe to be scanned even if symlink is not resolveable
if strings.HasPrefix(rootPath, "/proc/") {
root = rootPath
} else {
return nil, fmt.Errorf("eval %q: %w", rootPath, err)
}
}

err = filepath.WalkDir(root,
func(path string, info os.DirEntry, err error) error {
if err != nil {
logger.Errorf("error: %s: %s", path, err)
return nil
}
if info.IsDir() || strings.Contains(path, "/.git/") {
return nil
}

files = append(files, path)
return nil
})
return files, err
}

// cleanPath removes the temporary directory prefix from the path.
func cleanPath(path string, prefix string) string {
return strings.TrimPrefix(path, prefix)
}

// formatPath formats the path for display.
func formatPath(path string) string {
if strings.Contains(path, "\\") {
path = strings.ReplaceAll(path, "\\", "/")
}
return path
}
Loading

0 comments on commit c26df49

Please sign in to comment.