Simplify cache cleanup actions (#5215)

In our old approach, a workflow file contains a job that uploads the PR number as an artifact. While the PR is still open, the workflow_run triggered by it will download the artifact and use the information to clean up all except the last used cache associated with that original workflow. When a PR is merged or closed, there will be a post-pr workflow that uploads the PR number as an artifact and triggers a workflow_run that clean up all caches associated with the PR. The reason we did it this way was in the cache cleanup workflows, we did not find an easy way to get the number of the PR triggering them. This is not convenient because we have to add jobs uploading artifacts to workflow files. After some experiments, we have found a reliable way to find the PR number without using artifacts. The workflow_run's payload always contains the head SHA of the commit that triggers it, whether the PR comes from a fork or not. We can then use `gh pr list` to search for that head and obtain the PR number.
ECP-WarpX · Sep 5, 2024 · c9d79cc · c9d79cc
1 parent 6834a00
commit c9d79cc
Show file tree

Hide file tree

Showing 11 changed files with 18 additions and 135 deletions.
diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml
@@ -160,18 +160,3 @@ jobs:
         ulimit -c unlimited
 
         mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml
@@ -60,18 +60,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml
@@ -23,8 +23,11 @@ jobs:
 
           REPO=${{ github.repository }}
 
-          gh run download ${{ github.event.workflow_run.id }} -n pr_number
-          pr_number=`cat pr_number.txt`
+          # For debugging cat ${GITHUB_EVENT_PATH} to see the payload.
+
+          pr_head_sha=${{ github.event.workflow_run.head_sha }}
+          pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number')
+          echo "Post-PR cache cleanup for PR ${pr_number}"
           BRANCH=refs/pull/${pr_number}/merge
 
           # Setting this to not fail the workflow while deleting cache keys.

diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml
@@ -29,9 +29,12 @@ jobs:
           # Triggering workflow run name (e.g., LinuxClang)
           WORKFLOW_NAME="${{ github.event.workflow_run.name }}"
 
+          # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload.
+
           if [[ $EVENT == "pull_request" ]]; then
-            gh run download ${{ github.event.workflow_run.id }} -n pr_number
-            pr_number=`cat pr_number.txt`
+            pr_head_sha=${{ github.event.workflow_run.head_sha }}
+            pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number')
+            echo "Clean up cache for PR ${pr_number}"
             BRANCH=refs/pull/${pr_number}/merge
           else
             BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }}
@@ -54,6 +57,7 @@ jobs:
           IFS=$'\n'
           for j in $cached_jobs
           do
+            # Delete all entries except the last used one
             old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2)
             for k in $old_keys
             do

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -110,18 +110,3 @@ jobs:
         uses: github/codeql-action/upload-sarif@v3
         with:
           sarif_file: sarif-results/${{ matrix.language }}.sarif
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -203,18 +203,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml
@@ -127,18 +127,3 @@ jobs:
 
         ccache -s
         du -hs ~/.cache/ccache
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -194,18 +194,3 @@ jobs:
      #   python3 -m pip install --upgrade build packaging setuptools wheel
      #   PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel .
      #   python3 -m pip install *.whl
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml
@@ -85,18 +85,3 @@ jobs:
         export OMP_NUM_THREADS=1
 
         mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml
@@ -4,17 +4,13 @@ on:
     types:
       - closed
 
+# This workflow does not have the permission to clean up cache for PRs
+# originated from a fork. The purpose here is to trigger a workflow_run
+# cleanup-cache-postpr.yml that has the right permission.
+
 jobs:
-  cleanup:
+  noop:
     runs-on: ubuntu-latest
     steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1
+      - name: No OP
+        run: echo "This workflow is going to trigger CleanUpCachePostPR."
diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml
@@ -223,18 +223,3 @@ jobs:
       run: |
         export OMP_NUM_THREADS=1
         mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py
-
-  save_pr_number:
-    if: github.event_name == 'pull_request'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Save PR number
-        env:
-          PR_NUMBER: ${{ github.event.number }}
-        run: |
-          echo $PR_NUMBER > pr_number.txt
-      - uses: actions/upload-artifact@v4
-        with:
-          name: pr_number
-          path: pr_number.txt
-          retention-days: 1