From 58a89644088f7d25f72079ec86f00524d3c2a056 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sat, 6 May 2023 11:19:13 -0700 Subject: [PATCH] [chore]run govulncheck with multiple actions (#21524) Try to run govulncheck with multiple runners in parallel to avoid hitting memory limits. - Split the runner across the usual boundaries of receivers, exporters, connectors, etc groups. - Identify that the cmd/* go modules and root create a spike in memory that triggers OOM, those are now skipped. We create a pkg group to track all pkg/* go.mod. --------- Co-authored-by: Dmitrii Anoshin --- .github/workflows/build-and-test-windows.yml | 1 + .github/workflows/build-and-test.yml | 18 ++++++++++++++++-- Makefile | 14 ++++++++++++-- 3 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-test-windows.yml b/.github/workflows/build-and-test-windows.yml index 737acaeb9237..a614e8f93650 100644 --- a/.github/workflows/build-and-test-windows.yml +++ b/.github/workflows/build-and-test-windows.yml @@ -32,6 +32,7 @@ jobs: - exporter - extension - internal + - pkg - other runs-on: windows-latest if: ${{ github.actor != 'dependabot[bot]' && (contains(github.event.pull_request.labels.*.name, 'Run Windows') || github.event_name == 'push') }} diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index b6488c2bf766..4e44e00d70f1 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -69,6 +69,7 @@ jobs: - extension - connector - internal + - pkg - other runs-on: ubuntu-latest needs: [setup-environment] @@ -115,6 +116,18 @@ jobs: false fi govulncheck: + strategy: + fail-fast: false + matrix: + group: + - receiver-0 + - receiver-1 + - processor + - exporter + - extension + - connector + - internal + - pkg runs-on: ubuntu-latest timeout-minutes: 30 steps: @@ -123,7 +136,7 @@ jobs: - name: Setup Go uses: actions/setup-go@v4 with: - go-version: ~1.19.8 + go-version: ~1.19.9 - name: Cache Go id: go-cache uses: actions/cache@v3 @@ -136,7 +149,7 @@ jobs: if: steps.go-cache.outputs.cache-hit != 'true' run: make install-tools - name: Run `govulncheck` - run: make govulncheck + run: make -j2 gogovulncheck GROUP=${{ matrix.group }} checks: runs-on: ubuntu-latest needs: [setup-environment] @@ -207,6 +220,7 @@ jobs: - extension - connector - internal + - pkg - other runs-on: ubuntu-latest needs: [setup-environment] diff --git a/Makefile b/Makefile index 760dcd720243..03c970e633a4 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ FIND_MOD_ARGS=-type f -name "go.mod" TO_MOD_DIR=dirname {} \; | sort | grep -E '^./' EX_COMPONENTS=-not -path "./receiver/*" -not -path "./processor/*" -not -path "./exporter/*" -not -path "./extension/*" -not -path "./connector/*" EX_INTERNAL=-not -path "./internal/*" +EX_PKG=-not -path "./pkg/*" # NONROOT_MODS includes ./* dirs (excludes . dir) NONROOT_MODS := $(shell find . $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) @@ -32,8 +33,9 @@ EXPORTER_MODS := $(shell find ./exporter/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) EXTENSION_MODS := $(shell find ./extension/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) CONNECTOR_MODS := $(shell find ./connector/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) INTERNAL_MODS := $(shell find ./internal/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) -OTHER_MODS := $(shell find . $(EX_COMPONENTS) $(EX_INTERNAL) $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) $(PWD) -ALL_MODS := $(RECEIVER_MODS) $(PROCESSOR_MODS) $(EXPORTER_MODS) $(EXTENSION_MODS) $(CONNECTOR_MODS) $(INTERNAL_MODS) $(OTHER_MODS) +PKG_MODS := $(shell find ./pkg/* $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) +OTHER_MODS := $(shell find . $(EX_COMPONENTS) $(EX_INTERNAL) $(EX_PKG) $(FIND_MOD_ARGS) -exec $(TO_MOD_DIR) ) $(PWD) +ALL_MODS := $(RECEIVER_MODS) $(PROCESSOR_MODS) $(EXPORTER_MODS) $(EXTENSION_MODS) $(CONNECTOR_MODS) $(INTERNAL_MODS) $(PKG_MODS) $(OTHER_MODS) # find -exec dirname cannot be used to process multiple matching patterns FIND_INTEGRATION_TEST_MODS={ find . -type f -name "*integration_test.go" & find . -type f -name "*e2e_test.go" -not -path "./testbed/*"; } @@ -57,6 +59,7 @@ all-groups: @echo "\nextension: $(EXTENSION_MODS)" @echo "\nconnector: $(CONNECTOR_MODS)" @echo "\ninternal: $(INTERNAL_MODS)" + @echo "\npkg: $(PKG_MODS)" @echo "\nother: $(OTHER_MODS)" .PHONY: all @@ -106,6 +109,10 @@ gofmt: golint: $(MAKE) $(FOR_GROUP_TARGET) TARGET="lint" +.PHONY: gogovulncheck +gogovulncheck: + $(MAKE) $(FOR_GROUP_TARGET) TARGET="govulncheck" + .PHONY: goporto goporto: $(PORTO) $(PORTO) -w --include-internal --skip-dirs "^cmd$$" ./ @@ -193,6 +200,9 @@ for-connector-target: $(CONNECTOR_MODS) .PHONY: for-internal-target for-internal-target: $(INTERNAL_MODS) +.PHONY: for-pkg-target +for-pkg-target: $(PKG_MODS) + .PHONY: for-other-target for-other-target: $(OTHER_MODS)