ci: Shift (Docker) cache priming to request workflow (#37028)

Currently all triggered CI jobs check for required caches (and prime where required) This reduces the number of vms that get booted by checking once at the beginning of a run in the request phase Signed-off-by: Ryan Northey <[email protected]>
envoyproxy · Nov 8, 2024 · 46b954c · 46b954c
1 parent 232c19e
commit 46b954c
Show file tree

Hide file tree

Showing 23 changed files with 109 additions and 86 deletions.
diff --git a/.github/workflows/_load.yml b/.github/workflows/_load.yml
@@ -10,18 +10,11 @@ on:
         required: true
       app-key:
         required: true
-      lock-app-id:
-        required: true
-      lock-app-key:
-        required: true
 
     inputs:
       agent-ubuntu:
         type: string
         default: ubuntu-22.04
-      cache-docker:
-        type: boolean
-        default: true
       check-name:
         type: string
         required: true
@@ -168,30 +161,3 @@ jobs:
           | .summary = {title: .summary_title}
           | del(.request.message, .summary_title)
         print-result: ${{ fromJSON(env.CI_DEBUG || 'false') && true || false }}
-
-  cache:
-    secrets:
-      app-id: ${{ secrets.lock-app-id }}
-      app-key: ${{ secrets.lock-app-key }}
-    name: ${{ matrix.name || matrix.target }}
-    needs: request
-    uses: ./.github/workflows/_cache.yml
-    if: ${{ inputs.cache-docker && ! fromJSON(needs.request.outputs.skip) }}
-    with:
-      arch: ${{ matrix.arch }}
-      cache-suffix: ${{ matrix.cache-suffix }}
-      image-tag: ${{ fromJSON(needs.request.outputs.build-image).default }}
-      request: ${{ toJSON(needs.request.outputs) }}
-      runs-on: ${{ matrix.runs-on }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - target: docker-x64
-          name: Docker (x64)
-          arch: x64
-        - target: docker-arm64
-          name: Docker (arm64)
-          arch: arm64
-          cache-suffix: -arm64
-          runs-on: envoy-arm64-small
diff --git a/.github/workflows/_load_env.yml b/.github/workflows/_load_env.yml
@@ -106,7 +106,7 @@ jobs:
     secrets:
       app-id: ${{ secrets.lock-app-id }}
       app-key: ${{ secrets.lock-app-key }}
-    uses: ./.github/workflows/_cache.yml
+    uses: ./.github/workflows/_request_cache_docker.yml
     needs: request
     if: ${{ inputs.cache-docker }}
     with:

diff --git a/.github/workflows/_request.yml b/.github/workflows/_request.yml
@@ -10,6 +10,10 @@ on:
         required: true
       app-key:
         required: true
+      lock-app-id:
+        required: true
+      lock-app-key:
+        required: true
 
     # Defaults are set .github/config.yml on the `main` branch.
     inputs:
@@ -38,6 +42,7 @@ jobs:
       pull-requests: read
     outputs:
       env: ${{ steps.data.outputs.value }}
+      caches: ${{ steps.caches.outputs.value }}
       config: ${{ steps.config.outputs.config }}
     steps:
     - uses: envoyproxy/toolshed/gh-actions/[email protected]
@@ -69,6 +74,7 @@ jobs:
         started: ${{ steps.started.outputs.value }}
         token: ${{ secrets.GITHUB_TOKEN }}
         vars: ${{ toJSON(vars) }}
+
     - name: Request summary
       id: summary
       uses: envoyproxy/toolshed/gh-actions/github/env/[email protected]
@@ -101,7 +107,7 @@ jobs:
           | .env.config.envoy.icon as $icon
           | .link as $link
           | "\($icon) Request ([\($title)](\($link)))" as $linkedTitle
-          |  .summary as $summary
+          | .summary as $summary
           | .env
           | .summary = {
               $summary,
@@ -110,6 +116,42 @@ jobs:
               "linked-title": $linkedTitle}
           | del(.config.tables)
 
+    - name: Check Docker cache (x64)
+      id: cache-exists-docker-x64
+      uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+      with:
+        lookup-only: true
+        path: /tmp/cache
+        key: ${{ fromJSON(steps.data.outputs.value).request.build-image.default }}
+    - name: Check Docker cache (arm64)
+      id: cache-exists-docker-arm64
+      uses: actions/cache/restore@6849a6489940f00c2f30c0fb92c6274307ccb58a  # v4.1.2
+      with:
+        lookup-only: true
+        path: /tmp/cache
+        key: ${{ fromJSON(steps.data.outputs.value).request.build-image.default }}-arm64
+
+    - name: Caches
+      uses: envoyproxy/toolshed/gh-actions/[email protected]
+      id: caches
+      with:
+        input-format: yaml
+        input: |
+          docker:
+            x64: ${{ steps.cache-exists-docker-x64.outputs.cache-hit || 'false' }}
+            arm64: ${{ steps.cache-exists-docker-arm64.outputs.cache-hit || 'false' }}
+
+  cache:
+    if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }}
+    needs: incoming
+    uses: ./.github/workflows/_request_cache.yml
+    secrets:
+      app-id: ${{ secrets.lock-app-id }}
+      app-key: ${{ secrets.lock-app-key }}
+    with:
+      caches: ${{ needs.incoming.outputs.caches }}
+      env: ${{ needs.incoming.outputs.env }}
+
   checks:
     if: ${{ github.repository == 'envoyproxy/envoy' || vars.ENVOY_CI }}
     needs: incoming

diff --git a/.github/workflows/_request_cache.yml b/.github/workflows/_request_cache.yml
@@ -0,0 +1,45 @@
+name: Request/cache
+
+permissions:
+  contents: read
+
+on:
+  workflow_call:
+    secrets:
+      app-id:
+        required: true
+      app-key:
+        required: true
+
+    inputs:
+      env:
+        type: string
+        required: true
+      caches:
+        type: string
+        required: true
+
+
+jobs:
+  docker:
+    secrets:
+      app-id: ${{ secrets.app-id }}
+      app-key: ${{ secrets.app-key }}
+    name: Docker/${{ matrix.arch }}
+    uses: ./.github/workflows/_request_cache_docker.yml
+    with:
+      arch: ${{ matrix.arch }}
+      cache-suffix: ${{ matrix.cache-suffix }}
+      caches: ${{ inputs.caches }}
+      image-tag: ${{ fromJSON(inputs.env).request.build-image.default }}
+      runs-on: ${{ matrix.runs-on }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+        - target: docker-x64
+          arch: x64
+        - target: docker-arm64
+          arch: arm64
+          cache-suffix: -arm64
+          runs-on: envoy-arm64-small
diff --git a/.github/workflows/_cache.yml → .github/workflows/_request_cache_docker.yml b/.github/workflows/_cache.yml → .github/workflows/_request_cache_docker.yml
@@ -1,4 +1,4 @@
-name: Cache prime (docker)
+name: Request/cache (prime Docker)
 
 permissions:
   contents: read
@@ -11,18 +11,19 @@ on:
       app-key:
         required: true
     inputs:
+      caches:
+        type: string
+        required: true
+      image-tag:
+        type: string
+        required: true
+
       arch:
         type: string
         default: x64
       cache-suffix:
         type: string
         default:
-      image-tag:
-        type: string
-        required: true
-      request:
-        type: string
-        required: true
       runs-on:
         type: string
         default: ubuntu-24.04
@@ -47,6 +48,8 @@ on:
 jobs:
   docker:
     runs-on: ${{ inputs.runs-on || 'ubuntu-24.04' }}
+    name: "[${{ inputs.arch }}] Prime Docker cache"
+    if: ${{ ! fromJSON(inputs.caches).docker[inputs.arch] }}
     steps:
     - uses: envoyproxy/toolshed/gh-actions/[email protected]
       id: appauth

diff --git a/.github/workflows/_run.yml b/.github/workflows/_run.yml
@@ -238,6 +238,7 @@ jobs:
              "runs-on": $runsOn,
              "job-started": ${{ steps.started.outputs.value }}}
           | . * {$config, $check}
+
     - if: ${{ inputs.cache-build-image }}
       name: Restore Docker cache ${{ inputs.cache-build-image && format('({0})', inputs.cache-build-image) || '' }}
       uses: envoyproxy/toolshed/gh-actions/docker/cache/[email protected]

diff --git a/.github/workflows/envoy-checks.yml b/.github/workflows/envoy-checks.yml
@@ -30,8 +30,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/envoy-macos.yml b/.github/workflows/envoy-macos.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read
@@ -35,7 +33,6 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     uses: ./.github/workflows/_load.yml
     with:
-      cache-docker: false
       check-name: macos
 
   macos:

diff --git a/.github/workflows/envoy-prechecks.yml b/.github/workflows/envoy-prechecks.yml
@@ -28,8 +28,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/envoy-publish.yml b/.github/workflows/envoy-publish.yml
@@ -33,8 +33,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-android_build.yml b/.github/workflows/mobile-android_build.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-android_tests.yml b/.github/workflows/mobile-android_tests.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-asan.yml b/.github/workflows/mobile-asan.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-cc_tests.yml b/.github/workflows/mobile-cc_tests.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-coverage.yml b/.github/workflows/mobile-coverage.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-docs.yml b/.github/workflows/mobile-docs.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-format.yml b/.github/workflows/mobile-format.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-ios_build.yml b/.github/workflows/mobile-ios_build.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read
@@ -35,7 +33,6 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     uses: ./.github/workflows/_load.yml
     with:
-      cache-docker: false
       check-name: mobile-ios
 
   build:

diff --git a/.github/workflows/mobile-ios_tests.yml b/.github/workflows/mobile-ios_tests.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read
@@ -35,7 +33,6 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     uses: ./.github/workflows/_load.yml
     with:
-      cache-docker: false
       check-name: mobile-ios-tests
 
   tests:

diff --git a/.github/workflows/mobile-perf.yml b/.github/workflows/mobile-perf.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read

diff --git a/.github/workflows/mobile-release_validation.yml b/.github/workflows/mobile-release_validation.yml
@@ -25,8 +25,6 @@ jobs:
     secrets:
       app-key: ${{ secrets.ENVOY_CI_APP_KEY }}
       app-id: ${{ secrets.ENVOY_CI_APP_ID }}
-      lock-app-key: ${{ secrets.ENVOY_CI_MUTEX_APP_KEY }}
-      lock-app-id: ${{ secrets.ENVOY_CI_MUTEX_APP_ID }}
     permissions:
       actions: read
       contents: read
@@ -35,7 +33,6 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     uses: ./.github/workflows/_load.yml
     with:
-      cache-docker: false
       check-name: mobile-release-validation
 
   validate-swiftpm-example: