From 3bc38d6848281877ce2f04b01992450ef484b7fb Mon Sep 17 00:00:00 2001 From: Ryan Northey Date: Thu, 7 Nov 2024 07:47:31 +0000 Subject: [PATCH] ci: Shift Docker cache priming to request workflow Signed-off-by: Ryan Northey --- .github/workflows/_load.yml | 30 ---------- .github/workflows/_load_env.yml | 3 - .github/workflows/_request.yml | 45 ++++++++++++++- .github/workflows/_request_cache.yml | 57 +++++++++++++++++++ .../{_cache.yml => _request_cache_docker.yml} | 2 +- .../{_start.yml => _request_checks.yml} | 0 .github/workflows/request.yml | 16 +++--- 7 files changed, 110 insertions(+), 43 deletions(-) create mode 100644 .github/workflows/_request_cache.yml rename .github/workflows/{_cache.yml => _request_cache_docker.yml} (98%) rename .github/workflows/{_start.yml => _request_checks.yml} (100%) diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml index a0509997f049b..158c7ea04b425 100644 --- a/.github/workflows/_load.yml +++ b/.github/workflows/_load.yml @@ -19,9 +19,6 @@ on: agent-ubuntu: type: string default: ubuntu-22.04 - cache-docker: - type: boolean - default: true check-name: type: string required: true @@ -168,30 +165,3 @@ jobs: | .summary = {title: .summary_title} | del(.request.message, .summary_title) print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }} - - cache: - secrets: - app-id: ${{ secrets.lock-app-id }} - app-key: ${{ secrets.lock-app-key }} - name: ${{ matrix.name || matrix.target }} - needs: request - uses: ./.github/workflows/_cache.yml - if: ${{ inputs.cache-docker && ! fromJSON(needs.request.outputs.skip) }} - with: - arch: ${{ matrix.arch }} - cache-suffix: ${{ matrix.cache-suffix }} - image-tag: ${{ fromJSON(needs.request.outputs.build-image).default }} - request: ${{ toJSON(needs.request.outputs) }} - runs-on: ${{ matrix.runs-on }} - strategy: - fail-fast: false - matrix: - include: - - target: docker-x64 - name: Docker (x64) - arch: x64 - - target: docker-arm64 - name: Docker (arm64) - arch: arm64 - cache-suffix: -arm64 - runs-on: envoy-arm64-small diff --git a/.github/workflows/_load_env.yml b/.github/workflows/_load_env.yml index 95c6800ae8ae8..dd2c4d9479d0f 100644 --- a/.github/workflows/_load_env.yml +++ b/.github/workflows/_load_env.yml @@ -70,9 +70,6 @@ jobs: options: -r filter: | now - - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.3.1 - id: checkout - name: Checkout Envoy repository - name: Generate environment variables uses: envoyproxy/toolshed/gh-actions/envoy/ci/env@actions-v0.3.1 id: env diff --git a/.github/workflows/_request.yml b/.github/workflows/_request.yml index 8101409fe9a29..c262888dd91b0 100644 --- a/.github/workflows/_request.yml +++ b/.github/workflows/_request.yml @@ -10,9 +10,18 @@ on: required: true app-key: required: true + lock-app-id: + required: true + lock-app-key: + required: true # Defaults are set .github/config.yml on the `main` branch. inputs: + cache-bazel-hash-paths: + type: string + default: | + WORKSPACE + **/*.bzl config-file: type: string default: ./.github/config.yml @@ -55,6 +64,7 @@ jobs: branch: ${{ github.ref_name }} config: | fetch-depth: ${{ startsWith(github.event_name, 'pull_request') && 1 || 2 }} + path: requested # This step *LOOKS AT* the repo at the point requested # Its essential that this _job_ *MUST NOT EXECUTE ANY CODE FROM THE CHECKED OUT REPO* # *ALL* variables collected should be treated as untrusted and should be sanitized before @@ -69,6 +79,22 @@ jobs: started: ${{ steps.started.outputs.value }} token: ${{ secrets.GITHUB_TOKEN }} vars: ${{ toJSON(vars) }} + working-directory: requested + + - uses: envoyproxy/toolshed/gh-actions/github/checkout@actions-v0.3.1 + id: checkout-target + name: Checkout Envoy repository (target branch) + with: + branch: ${{ fromJSON(steps.env.outputs.data).request.target-branch }} + config: | + fetch-depth: 1 + path: target + - uses: envoyproxy/toolshed/gh-actions/hashfiles@actions-v0.3.1 + id: bazel-cache-hash + with: + files: ${{ inputs.cache-bazel-hash-paths }} + working-directory: target + - name: Request summary id: summary uses: envoyproxy/toolshed/gh-actions/github/env/summary@actions-v0.3.1 @@ -91,18 +117,23 @@ jobs: id: data with: input: | + cache: + bazel: ${{ steps.bazel-cache-hash.outputs.value }} env: ${{ steps.env.outputs.data }} title: ${{ steps.summary.outputs.title }} link: ${{ format('https://github.com/{0}/actions/runs/{1}', github.repository, github.run_id) }} summary: ${{ steps.summary.outputs.summary }} input-format: yaml + print-result: true filter: | .title as $title | .env.config.envoy.icon as $icon + | .cache as $cache | .link as $link | "\($icon) Request ([\($title)](\($link)))" as $linkedTitle - | .summary as $summary + | .summary as $summary | .env + | .config.ci.cache = $cache | .summary = { $summary, $title, @@ -110,10 +141,20 @@ jobs: "linked-title": $linkedTitle} | del(.config.tables) + cache: + if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} + needs: incoming + uses: ./.github/workflows/_request_cache.yml + secrets: + app-id: ${{ secrets.lock-app-id }} + app-key: ${{ secrets.lock-app-key }} + with: + env: ${{ needs.incoming.outputs.env }} + checks: if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }} needs: incoming - uses: ./.github/workflows/_start.yml + uses: ./.github/workflows/_request_checks.yml secrets: app-id: ${{ secrets.app-id }} app-key: ${{ secrets.app-key }} diff --git a/.github/workflows/_request_cache.yml b/.github/workflows/_request_cache.yml new file mode 100644 index 0000000000000..f0dcc9433e1a1 --- /dev/null +++ b/.github/workflows/_request_cache.yml @@ -0,0 +1,57 @@ +name: Request/cache + +permissions: + contents: read + +on: + workflow_call: + secrets: + app-id: + required: true + app-key: + required: true + + inputs: + env: + type: string + required: true + + cache-docker: + type: boolean + default: true + + +jobs: + debug: + runs-on: ubuntu-22.04 + steps: + - uses: envoyproxy/toolshed/gh-actions/jq@actions-v0.3.1 + with: + input: ${{ inputs.env }} + print-result: true + + docker: + secrets: + app-id: ${{ secrets.app-id }} + app-key: ${{ secrets.app-key }} + name: ${{ matrix.name || matrix.target }} + uses: ./.github/workflows/_request_cache_docker.yml + # if: ${{ inputs.cache-docker && ! fromJSON(needs.request.outputs.skip) }} + with: + arch: ${{ matrix.arch }} + cache-suffix: ${{ matrix.cache-suffix }} + image-tag: ${{ fromJSON(inputs.env).request.build-image.default }} + request: ${{ toJSON(fromJSON(inputs.env).request) }} + runs-on: ${{ matrix.runs-on }} + strategy: + fail-fast: false + matrix: + include: + - target: docker-x64 + name: Docker (x64) + arch: x64 + - target: docker-arm64 + name: Docker (arm64) + arch: arm64 + cache-suffix: -arm64 + runs-on: envoy-arm64-small diff --git a/.github/workflows/_cache.yml b/.github/workflows/_request_cache_docker.yml similarity index 98% rename from .github/workflows/_cache.yml rename to .github/workflows/_request_cache_docker.yml index 09d86448ef900..75d34c9671e01 100644 --- a/.github/workflows/_cache.yml +++ b/.github/workflows/_request_cache_docker.yml @@ -1,4 +1,4 @@ -name: Cache prime (docker) +name: Request/Cache prime (docker) permissions: contents: read diff --git a/.github/workflows/_start.yml b/.github/workflows/_request_checks.yml similarity index 100% rename from .github/workflows/_start.yml rename to .github/workflows/_request_checks.yml diff --git a/.github/workflows/request.yml b/.github/workflows/request.yml index cc37388e09b47..da826858e7426 100644 --- a/.github/workflows/request.yml +++ b/.github/workflows/request.yml @@ -24,13 +24,6 @@ concurrency: jobs: request: - # For branches this can be pinned to a specific version if required - # NB: `uses` cannot be dynamic so it _must_ be hardcoded anywhere it is read - uses: envoyproxy/envoy/.github/workflows/_request.yml@main - if: >- - ${{ github.repository == 'envoyproxy/envoy' - || (vars.ENVOY_CI && github.event_name != 'schedule') - || (vars.ENVOY_SCHEDULED_CI && github.event_name == 'schedule') }} permissions: actions: read contents: read @@ -41,3 +34,12 @@ jobs: # these are required to start checks app-key: ${{ secrets.ENVOY_CI_APP_KEY }} app-id: ${{ secrets.ENVOY_CI_APP_ID }} + lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }} + lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }} + # For branches this can be pinned to a specific version if required + # NB: `uses` cannot be dynamic so it _must_ be hardcoded anywhere it is read + uses: envoyproxy/envoy-ci-staging/.github/workflows/_request.yml@main + if: >- + ${{ github.repository == 'envoyproxy/envoy' + || (vars.ENVOY_CI && github.event_name != 'schedule') + || (vars.ENVOY_SCHEDULED_CI && github.event_name == 'schedule') }}