From c9d79ccf2de64b00f6be02432ad33bae641c50a8 Mon Sep 17 00:00:00 2001 From: Weiqun Zhang Date: Wed, 4 Sep 2024 19:32:22 -0500 Subject: [PATCH] Simplify cache cleanup actions (#5215) In our old approach, a workflow file contains a job that uploads the PR number as an artifact. While the PR is still open, the workflow_run triggered by it will download the artifact and use the information to clean up all except the last used cache associated with that original workflow. When a PR is merged or closed, there will be a post-pr workflow that uploads the PR number as an artifact and triggers a workflow_run that clean up all caches associated with the PR. The reason we did it this way was in the cache cleanup workflows, we did not find an easy way to get the number of the PR triggering them. This is not convenient because we have to add jobs uploading artifacts to workflow files. After some experiments, we have found a reliable way to find the PR number without using artifacts. The workflow_run's payload always contains the head SHA of the commit that triggers it, whether the PR comes from a fork or not. We can then use `gh pr list` to search for that head and obtain the PR number. --- .github/workflows/clang_sanitizers.yml | 15 --------------- .github/workflows/clang_tidy.yml | 15 --------------- .github/workflows/cleanup-cache-postpr.yml | 7 +++++-- .github/workflows/cleanup-cache.yml | 8 ++++++-- .github/workflows/codeql.yml | 15 --------------- .github/workflows/cuda.yml | 15 --------------- .github/workflows/hip.yml | 15 --------------- .github/workflows/intel.yml | 15 --------------- .github/workflows/macos.yml | 15 --------------- .github/workflows/post-pr.yml | 18 +++++++----------- .github/workflows/ubuntu.yml | 15 --------------- 11 files changed, 18 insertions(+), 135 deletions(-) diff --git a/.github/workflows/clang_sanitizers.yml b/.github/workflows/clang_sanitizers.yml index ef005cbbc72..8efcdc9a431 100644 --- a/.github/workflows/clang_sanitizers.yml +++ b/.github/workflows/clang_sanitizers.yml @@ -160,18 +160,3 @@ jobs: ulimit -c unlimited mpirun -n 2 ../../../build_EB/bin/warpx.2d inputs_2d warpx.serialize_initial_conditions = 0 - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 96c7337a3f2..5a4f83f01f1 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -60,18 +60,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cleanup-cache-postpr.yml b/.github/workflows/cleanup-cache-postpr.yml index 9a2ffb0f61a..5e9a70cd5a4 100644 --- a/.github/workflows/cleanup-cache-postpr.yml +++ b/.github/workflows/cleanup-cache-postpr.yml @@ -23,8 +23,11 @@ jobs: REPO=${{ github.repository }} - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + # For debugging cat ${GITHUB_EVENT_PATH} to see the payload. + + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --state all --search $pr_head_sha --json number --jq '.[0].number') + echo "Post-PR cache cleanup for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge # Setting this to not fail the workflow while deleting cache keys. diff --git a/.github/workflows/cleanup-cache.yml b/.github/workflows/cleanup-cache.yml index bd1a518acf4..3abe232b879 100644 --- a/.github/workflows/cleanup-cache.yml +++ b/.github/workflows/cleanup-cache.yml @@ -29,9 +29,12 @@ jobs: # Triggering workflow run name (e.g., LinuxClang) WORKFLOW_NAME="${{ github.event.workflow_run.name }}" + # For debugging, cat ${GITHUB_EVENT_PATH} to see the payload. + if [[ $EVENT == "pull_request" ]]; then - gh run download ${{ github.event.workflow_run.id }} -n pr_number - pr_number=`cat pr_number.txt` + pr_head_sha=${{ github.event.workflow_run.head_sha }} + pr_number=$(gh pr list --search $pr_head_sha --json number --jq '.[0].number') + echo "Clean up cache for PR ${pr_number}" BRANCH=refs/pull/${pr_number}/merge else BRANCH=refs/heads/${{ github.event.workflow_run.head_branch }} @@ -54,6 +57,7 @@ jobs: IFS=$'\n' for j in $cached_jobs do + # Delete all entries except the last used one old_keys=$(gh actions-cache list -L 100 -R $REPO -B $BRANCH --key "${j}-git-" --sort last-used | cut -f 1 | tail -n +2) for k in $old_keys do diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index bc0bee545cc..5c36b9d9f21 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -110,18 +110,3 @@ jobs: uses: github/codeql-action/upload-sarif@v3 with: sarif_file: sarif-results/${{ matrix.language }}.sarif - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index 9554e1faa3a..010ce8090ac 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -203,18 +203,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 8b89e3dc4d0..ba537e776d4 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -127,18 +127,3 @@ jobs: ccache -s du -hs ~/.cache/ccache - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml index 8a7867fcbc8..485a5229c6a 100644 --- a/.github/workflows/intel.yml +++ b/.github/workflows/intel.yml @@ -194,18 +194,3 @@ jobs: # python3 -m pip install --upgrade build packaging setuptools wheel # PYWARPX_LIB_DIR=$PWD/build_sp/lib/site-packages/pywarpx/ python3 -m pip wheel . # python3 -m pip install *.whl - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index dfd32f459f0..124d26fa7f7 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -85,18 +85,3 @@ jobs: export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 diff --git a/.github/workflows/post-pr.yml b/.github/workflows/post-pr.yml index 2768ef376cc..5f0b1534970 100644 --- a/.github/workflows/post-pr.yml +++ b/.github/workflows/post-pr.yml @@ -4,17 +4,13 @@ on: types: - closed +# This workflow does not have the permission to clean up cache for PRs +# originated from a fork. The purpose here is to trigger a workflow_run +# cleanup-cache-postpr.yml that has the right permission. + jobs: - cleanup: + noop: runs-on: ubuntu-latest steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1 + - name: No OP + run: echo "This workflow is going to trigger CleanUpCachePostPR." diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 05e53883534..bf6652e9c69 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -223,18 +223,3 @@ jobs: run: | export OMP_NUM_THREADS=1 mpirun -n 2 Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py - - save_pr_number: - if: github.event_name == 'pull_request' - runs-on: ubuntu-latest - steps: - - name: Save PR number - env: - PR_NUMBER: ${{ github.event.number }} - run: | - echo $PR_NUMBER > pr_number.txt - - uses: actions/upload-artifact@v4 - with: - name: pr_number - path: pr_number.txt - retention-days: 1