Another attempt to make jobs non-concurrent on a per-PR basis (#1974)

* Another attempt to make jobs non-concurrent on a per-PR basis * Move concurrency into the reusable job
rerun-io · Apr 26, 2023 · e4814bc · e4814bc
1 parent d686ca2
commit e4814bc
Show file tree

Hide file tree

Showing 14 changed files with 128 additions and 66 deletions.
diff --git a/.github/workflows/manual_build_wheels_for_pr.yml b/.github/workflows/manual_build_wheels_for_pr.yml
@@ -32,6 +32,7 @@ jobs:
     name: 'Linux: Build/Test Wheels'
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PLATFORM: linux
       WHEEL_ARTIFACT_NAME: linux-wheel
       RRD_ARTIFACT_NAME: linux-rrd
@@ -42,6 +43,7 @@ jobs:
     name: 'Windows: Build/Test Wheels'
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PLATFORM: windows
       WHEEL_ARTIFACT_NAME: windows-wheel
       RRD_ARTIFACT_NAME: ''
@@ -52,6 +54,7 @@ jobs:
     name: 'Macos-Arm: Build/Test Wheels'
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PLATFORM: macos-arm
       WHEEL_ARTIFACT_NAME: macos-arm-wheel
       RRD_ARTIFACT_NAME: ''
@@ -62,6 +65,7 @@ jobs:
     name: 'Macos-Intel: Build/Test Wheels'
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PLATFORM: macos-intel
       WHEEL_ARTIFACT_NAME: 'macos-intel-wheel'
       RRD_ARTIFACT_NAME: ''
@@ -72,6 +76,7 @@ jobs:
     needs: [build-linux]
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       WHEEL_ARTIFACT_NAME: linux-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -81,6 +86,7 @@ jobs:
     needs: [build-linux, build-windows]
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       WHEEL_ARTIFACT_NAME: windows-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -90,6 +96,7 @@ jobs:
     needs: [build-linux, build-macos-arm]
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       WHEEL_ARTIFACT_NAME: macos-arm-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -99,6 +106,7 @@ jobs:
     needs: [build-linux, build-macos-intel]
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       WHEEL_ARTIFACT_NAME: macos-intel-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -107,12 +115,15 @@ jobs:
     name: 'Generate Pip Index'
     needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel]
     uses: ./.github/workflows/reusable_pip_index.yml
+    with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
     secrets: inherit
 
   update-pr-summary:
     name: 'Update PR Summary'
     needs: [check-for-pr, upload-wheels-linux, upload-wheels-windows, upload-wheels-macos-arm, upload-wheels-macos-intel]
     uses: ./.github/workflows/reusable_pr_summary.yml
     with:
+      CONCURRENCY: manual-wheels-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER}}
     secrets: inherit
diff --git a/.github/workflows/manual_dispatch.yml b/.github/workflows/manual_dispatch.yml
@@ -72,6 +72,7 @@ jobs:
     if: ${{ github.event.inputs.CHECKS == 'true' }}
     uses: ./.github/workflows/reusable_checks.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
     secrets: inherit
 
@@ -82,6 +83,7 @@ jobs:
     if: ${{ github.event.inputs.DEPLOY_DOCS == 'true' }}
     uses: ./.github/workflows/reusable_deploy_docs.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       PY_DOCS_VERSION_NAME: "test"
       UPDATE_LATEST: false
     secrets: inherit
@@ -90,6 +92,8 @@ jobs:
     name: Benchmarks
     if: ${{ github.event.inputs.BENCHES == 'true' }}
     uses: ./.github/workflows/reusable_bench.yml
+    with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
     secrets: inherit
 
   min-test-wheel:
@@ -98,6 +102,7 @@ jobs:
     if: ${{ (github.event.inputs.MIN_TEST_WHEEL == 'true') || ((github.event.inputs.UPLOAD_GCLOUD == 'true') && ( github.event.inputs.BUILD_WEB == 'true') ) }}
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
       PLATFORM: linux
       MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module'
@@ -113,6 +118,7 @@ jobs:
 
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
       PLATFORM: linux
       WHEEL_ARTIFACT_NAME: linux-wheel
@@ -125,6 +131,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') }}
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
       PLATFORM: windows
       WHEEL_ARTIFACT_NAME: windows-wheel
@@ -137,6 +144,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') }}
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
       PLATFORM: macos-arm
       WHEEL_ARTIFACT_NAME: macos-arm-wheel
@@ -149,6 +157,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') }}
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       SAVE_CACHE: ${{ github.event.inputs.SAVE_CACHE == 'true' }}
       PLATFORM: macos-intel
       WHEEL_ARTIFACT_NAME: 'macos-intel-wheel'
@@ -163,6 +172,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'linux') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }}
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       WHEEL_ARTIFACT_NAME: linux-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -173,6 +183,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'windows') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }}
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       WHEEL_ARTIFACT_NAME: windows-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -183,6 +194,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-arm') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }}
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       WHEEL_ARTIFACT_NAME: macos-arm-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -193,6 +205,7 @@ jobs:
     if: ${{ contains(github.event.inputs.WHEEL_PLATFORMS, 'macos-intel') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }}
     uses: ./.github/workflows/reusable_upload_wheels.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       WHEEL_ARTIFACT_NAME: macos-intel-wheel
       RRD_ARTIFACT_NAME: linux-rrd
     secrets: inherit
@@ -202,6 +215,7 @@ jobs:
     if: ${{ github.event.inputs.BUILD_WEB == 'true'}}
     uses: ./.github/workflows/reusable_build_web.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       RELEASE_VERSION: ${{ github.event.inputs.RELEASE_VERSION }}
     secrets: inherit
 
@@ -211,6 +225,7 @@ jobs:
     if: ${{ (github.event.inputs.BUILD_WEB == 'true') && (github.event.inputs.UPLOAD_GCLOUD == 'true') }}
     uses: ./.github/workflows/reusable_upload_web.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       MARK_PRERELEASE_FOR_MAINLINE: false
       MARK_TAGGED_VERSION: false
       RRD_ARTIFACT_NAME: linux-rrd-fast
@@ -250,5 +265,6 @@ jobs:
     if: ${{ github.event.inputs.UPDATE_PR_SUMMARY == 'true' }}
     uses: ./.github/workflows/reusable_pr_summary.yml
     with:
+      CONCURRENCY: manual-dispatch-${{ github.run_id}}
       PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER }}
     secrets: inherit
diff --git a/.github/workflows/manual_run_bench.yml b/.github/workflows/manual_run_bench.yml
@@ -16,14 +16,6 @@ on:
 
 jobs:
 
-  run-benches:
-    name: Benchmarks
-    uses: ./.github/workflows/reusable_bench.yml
-    with:
-      BENCH_NAME: ${{ github.event.inputs.BENCH_NAME }}
-      COMPARE_TO: ${{ github.event.inputs.COMPARE_TO }}
-    secrets: inherit
-
   check-for-pr:
     runs-on: ubuntu-latest
     outputs:
@@ -46,10 +38,22 @@ jobs:
             echo "PR_NUMBER=$pr_number" >> "$GITHUB_OUTPUT"
           fi
 
+  run-benches:
+    name: Benchmarks
+    needs: [check-for-pr]
+    uses: ./.github/workflows/reusable_bench.yml
+    with:
+      CONCURRENCY: manual-bench-${{ needs.check-for-pr.outputs.PR_NUMBER }}
+      BENCH_NAME: ${{ github.event.inputs.BENCH_NAME }}
+      COMPARE_TO: ${{ github.event.inputs.COMPARE_TO }}
+    secrets: inherit
+
+
   update-pr-summary:
     name: 'Update PR Summary'
     needs: [check-for-pr, run-benches]
     uses: ./.github/workflows/reusable_pr_summary.yml
     with:
+      CONCURRENCY: manual-bench-${{ needs.check-for-pr.outputs.PR_NUMBER }}
       PR_NUMBER: ${{ needs.check-for-pr.outputs.PR_NUMBER}}
     secrets: inherit
diff --git a/.github/workflows/on_pull_request.yml b/.github/workflows/on_pull_request.yml
@@ -11,52 +11,44 @@ jobs:
   checks:
     name: Checks
     uses: ./.github/workflows/reusable_checks.yml
+    with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
     secrets: inherit
-    concurrency:
-      group: pr-checks-${{ github.event.pull_request.number }}
-      cancel-in-progress: true
 
   min-test-wheel:
     name: 'Minimum Wheel'
     uses: ./.github/workflows/reusable_build_and_test_wheels.yml
     with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
       SAVE_CACHE: false
       PLATFORM: linux
       MATURIN_FEATURE_FLAGS: '--no-default-features --features extension-module'
       WHEEL_ARTIFACT_NAME: '' # the min-test wheel isn't used for anything
       RRD_ARTIFACT_NAME: linux-rrd-fast
     secrets: inherit
-    concurrency:
-      group: pr-min-test-wheel-${{ github.event.pull_request.number }}
-      cancel-in-progress: true
 
   build-web:
     name: 'Build Web'
     uses: ./.github/workflows/reusable_build_web.yml
+    with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
     secrets: inherit
-    concurrency:
-      group: pr-build-web-${{ github.event.pull_request.number }}
-      cancel-in-progress: true
 
   upload-web:
     name: 'Upload Web'
     needs: [min-test-wheel, build-web]
     uses: ./.github/workflows/reusable_upload_web.yml
     with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
       RRD_ARTIFACT_NAME: linux-rrd-fast
       UPLOAD_COMMIT_OVERRIDE: ${{ github.event.pull_request.head.sha }}
     secrets: inherit
-    concurrency:
-      group: pr-upload-web-${{ github.event.pull_request.number }}
-      cancel-in-progress: true
 
   save-pr-summary:
     name: 'Save PR Summary'
     needs: [upload-web]
     uses: ./.github/workflows/reusable_pr_summary.yml
     with:
+      CONCURRENCY: pr-${{ github.event.pull_request.number }}
       PR_NUMBER: ${{ github.event.pull_request.number }}
     secrets: inherit
-    concurrency:
-      group: pr-save-pr-${{ github.event.pull_request.number }}
-      cancel-in-progress: true