From fe46425b5273e9243ddfa6ddc1e6194e439ed547 Mon Sep 17 00:00:00 2001 From: Spencer Schrock Date: Fri, 21 Jul 2023 14:46:49 -0700 Subject: [PATCH] :seedling: Add separate cache for long-running tests (#3293) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add separate cache for unit tests. Signed-off-by: Spencer Schrock * share cache with gitlab tests too. Signed-off-by: Spencer Schrock * share cache with github integration tests. Signed-off-by: Spencer Schrock * explicitly download modules in unit test job Signed-off-by: Spencer Schrock * checkout needs to be before the go.mod is read. Signed-off-by: Spencer Schrock * checkout needs to be before the go.sum files are hashed. Signed-off-by: Spencer Schrock --------- Signed-off-by: Spencer Schrock Signed-off-by: André Backman --- .github/workflows/gitlab.yml | 24 +++++++++++++++---- .github/workflows/integration.yml | 36 ++++++++++++++++++++-------- .github/workflows/main.yml | 40 ++++++++++++++++--------------- 3 files changed, 67 insertions(+), 33 deletions(-) diff --git a/.github/workflows/gitlab.yml b/.github/workflows/gitlab.yml index 964c47f2834..6ba61e9405f 100644 --- a/.github/workflows/gitlab.yml +++ b/.github/workflows/gitlab.yml @@ -24,6 +24,9 @@ on: branches: - main +env: + GO_VERSION_FILE: go.mod # no good way of getting a mutual version between go.mod and tools/go.mod + jobs: gitlab-integration-trusted: runs-on: ubuntu-latest @@ -33,17 +36,30 @@ jobs: uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - name: Clone the code uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} # head SHA if PR, else fallback to push SHA - - - name: setup-go + - name: Setup Go uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: - go-version: '1.19' + go-version-file: ${{ env.GO_VERSION_FILE }} check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 + with: + path: | + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-tests- - name: Prepare test env run: | diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index f2e88119a0b..c4e70ccf88d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -23,6 +23,9 @@ on: permissions: contents: read +env: + GO_VERSION_FILE: go.mod # no good way of getting a mutual version between go.mod and tools/go.mod + jobs: approve: runs-on: ubuntu-latest @@ -41,27 +44,40 @@ jobs: needs: [approve] steps: - name: Harden Runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1 + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - - - name: pull_request actions/checkout - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4 + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 with: ref: ${{ github.event.pull_request.head.sha }} - - - name: setup-go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0 + - name: Setup Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 with: - go-version: '1.19' + go-version-file: ${{ env.GO_VERSION_FILE }} check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" + - name: Cache builds + # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds + uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 + with: + path: | + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go-tests- - name: Prepare test env run: | go mod download - name: Run GITHUB_TOKEN E2E #using retry because the GitHub token is being throttled. - uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd + uses: nick-invision/retry@943e742917ac94714d2f408a0e8320f2d1fcafcd # v2.8.3 env: GITHUB_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: @@ -71,7 +87,7 @@ jobs: command: make e2e-gh-token - name: codecov - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 2.1.0 + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4 with: files: "*e2e-coverage.out" verbose: true diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 16457560133..a995caa1c77 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -37,36 +37,38 @@ jobs: contents: read steps: - name: Harden Runner - uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v1 + uses: step-security/harden-runner@55d479fb1c5bcad5a4f9099a5d9f37c8857b2845 # v2.4.1 with: egress-policy: audit # TODO: change to 'egress-policy: block' after couple of runs - + - name: Clone the code + uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3 + - name: Setup Go + uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1 + with: + go-version: ${{ env.GO_VERSION }} + check-latest: true + cache: false # we manually manage caches below + - id: go-cache-paths + run: | + echo "go-build=$(go env GOCACHE)" >> "$GITHUB_OUTPUT" + echo "go-mod=$(go env GOMODCACHE)" >> "$GITHUB_OUTPUT" - name: Cache builds # https://github.com/mvdan/github-actions-golang#how-do-i-set-up-caching-between-builds uses: actions/cache@88522ab9f39a2ea568f7027eddc7d8d8bc9d59c8 #v3.3.1 with: path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + ${{ steps.go-cache-paths.outputs.go-build }} + ${{ steps.go-cache-paths.outputs.go-mod }} + key: ${{ runner.os }}-go-tests-${{ hashFiles('**/go.sum') }} restore-keys: | - ${{ runner.os }}-go- - - name: Clone the code - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v2.3.4 - with: - fetch-depth: 0 - - name: Setup Go - uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v2.2.0 - with: - go-version: ${{ env.GO_VERSION }} - check-latest: true - cache: true + ${{ runner.os }}-go-tests- + - name: Prepare test env + run: | + go mod download - name: Run unit-tests run: make unit-test - name: Upload codecoverage - uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 2.1.0 + uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # 3.1.4 with: files: ./unit-coverage.out verbose: true