Preflight workflow update (#5771)

- New and updated vars - New runner vars - Runners rename (see paritytech/ci_cd#1039) --------- Co-authored-by: Oliver Tale-Yazdi <[email protected]>
paritytech · Sep 24, 2024 · c987da3 · c987da3
1 parent 1790e42
commit c987da3
Show file tree

Hide file tree

Showing 4 changed files with 114 additions and 24 deletions.
diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml
@@ -26,36 +26,77 @@ on:
       IMAGE:
         value: ${{ jobs.preflight.outputs.IMAGE }}
         description: "CI image"
+
+      # Runners
+      # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners
       RUNNER:
         value: ${{ jobs.preflight.outputs.RUNNER }}
         description: |
-          Runner name.
+          Main runner for resource-intensive tasks
+          By default we use spot machines that can be terminated at any time.
+          Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
+      RUNNER_OLDLINUX:
+        value: ${{ jobs.preflight.outputs.RUNNER_OLDLINUX }}
+        description: |
+          parity-oldlinux
           By default we use spot machines that can be terminated at any time.
           Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
-      OLDLINUXRUNNER:
-        value: ${{ jobs.preflight.outputs.OLDLINUXRUNNER }}
+      RUNNER_DEFAULT:
+        value: ${{ jobs.preflight.outputs.RUNNER_DEFAULT }}
+        description: "Relatively lightweight runner. When `ubuntu-latest` is not enough"
+      RUNNER_WEIGHTS:
+        value: ${{ jobs.preflight.outputs.RUNNER_WEIGHTS }}
+      RUNNER_BENCHMARK:
+        value: ${{ jobs.preflight.outputs.RUNNER_BENCHMARK }}
+      RUNNER_MACOS:
+        value: ${{ jobs.preflight.outputs.RUNNER_MACOS }}
+
+      # Vars
+      SOURCE_REF_SLUG:
+        value: ${{ jobs.preflight.outputs.SOURCE_REF_SLUG }}
+        description: "Name of the current branch for `push` or source branch for `pull_request` with `/` replaced by `_`. Does not exists in merge_group"
+      REF_SLUG:
+        value: ${{ jobs.preflight.outputs.REF_SLUG }}
+        description: |
+          Name of the current revision (depending on the event) with `/` replaced by `_`, e.g:
+          push - master
+          pull_request - 49_merge
+          merge_group - gh-readonly-queue_master_pr-49-38d43798a986430231c828b2c762997f818ac012
 
-      SOURCE_REF_NAME:
-        value: ${{ jobs.preflight.outputs.SOURCE_REF_NAME }}
-        description: "Name of the current branch for `push` or source branch for `pull_request`"
       COMMIT_SHA:
         value: ${{ jobs.preflight.outputs.COMMIT_SHA }}
-        description: "Sha of the current commit for `push` or head of the source branch for `pull_request`"
+        description: "Sha of the current revision"
+      COMMIT_SHA_SHORT:
+        value: ${{ jobs.preflight.outputs.COMMIT_SHA_SHORT }}
+        description: "Sha of the current revision, 8-symbols long"
 
 jobs:
+
+  #
+  #
+  #
   preflight:
     runs-on: ubuntu-latest
     outputs:
       changes_rust: ${{ steps.set_changes.outputs.rust_any_changed || steps.set_changes.outputs.currentWorkflow_any_changed }}
       changes_currentWorkflow: ${{ steps.set_changes.outputs.currentWorkflow_any_changed }}
 
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+
+      # Runners
+      # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners
       RUNNER: ${{ steps.set_runner.outputs.RUNNER }}
-      OLDLINUXRUNNER: ${{ steps.set_runner.outputs.OLDLINUXRUNNER }}
+      RUNNER_OLDLINUX: ${{ steps.set_runner.outputs.RUNNER_OLDLINUX }}
+      RUNNER_DEFAULT: ${{ steps.set_runner.outputs.RUNNER_DEFAULT }}
+      RUNNER_WEIGHTS: ${{ steps.set_runner.outputs.RUNNER_WEIGHTS }}
+      RUNNER_BENCHMARK: ${{ steps.set_runner.outputs.RUNNER_BENCHMARK }}
+      RUNNER_MACOS: ${{ steps.set_runner.outputs.RUNNER_MACOS }}
 
-      SOURCE_REF_NAME: ${{ steps.set_vars.outputs.SOURCE_REF_NAME }}
-      COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }}
+      SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }}
+      REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }}
 
+      COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }}
+      COMMIT_SHA_SHORT: ${{ steps.set_vars.outputs.COMMIT_SHA_SHORT }}
 
     steps:
 
@@ -64,7 +105,8 @@ jobs:
       #
       # Set changes
       #
-      - id: current_file
+      - name: Current file
+        id: current_file
         shell: bash
         run: |
           echo "currentWorkflowFile=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
@@ -98,34 +140,82 @@ jobs:
       # By default we use spot machines that can be terminated at any time.
       # Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
       #
-      - id: set_runner
+      - name: Set runner
+        id: set_runner
         shell: bash
         run: |
+          echo "RUNNER_DEFAULT=parity-default" >> $GITHUB_OUTPUT
+          echo "RUNNER_WEIGHTS=parity-weights" >> $GITHUB_OUTPUT
+          echo "RUNNER_BENCHMARK=parity-benchmark" >> $GITHUB_OUTPUT
+          echo "RUNNER_MACOS=parity-macos" >> $GITHUB_OUTPUT
+          #
           # Run merge queues on persistent runners
           if [[ $GITHUB_REF_NAME == *"gh-readonly-queue"* ]]; then
-            echo "RUNNER=arc-runners-polkadot-sdk-beefy-persistent" >> $GITHUB_OUTPUT
-            echo "OLDLINUXRUNNER=oldlinux-persistent" >> $GITHUB_OUTPUT
+            echo "RUNNER=parity-large-persistent" >> $GITHUB_OUTPUT
+            echo "RUNNER_OLDLINUX=parity-oldlinux-persistent" >> $GITHUB_OUTPUT
           else
-            echo "RUNNER=arc-runners-polkadot-sdk-beefy" >> $GITHUB_OUTPUT
-            echo "OLDLINUXRUNNER=oldlinux" >> $GITHUB_OUTPUT
+            echo "RUNNER=parity-large" >> $GITHUB_OUTPUT
+            echo "RUNNER_OLDLINUX=parity-oldlinux" >> $GITHUB_OUTPUT
           fi
 
       #
       # Set vars
       #
-      - id: set_vars
+      - name: Set vars
+        id: set_vars
         shell: bash
         run: |
-          export BRANCH_NAME=${{ github.head_ref || github.ref_name }}
-          echo "SOURCE_REF_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_OUTPUT
-          echo "COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_OUTPUT
+          export SOURCE_REF_NAME=${{ github.head_ref || github.ref_name }}
+          echo "SOURCE_REF_SLUG=${SOURCE_REF_NAME//\//_}" >> $GITHUB_OUTPUT
+          #
+          export COMMIT_SHA=${{ github.sha }}
+          echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_OUTPUT
+          echo "COMMIT_SHA_SHORT=${COMMIT_SHA:0:8}" >> $GITHUB_OUTPUT
+          #
+          export REF_NAME=${{ github.ref_name }}
+          echo "REF_SLUG=${REF_NAME//\//_}" >> $GITHUB_OUTPUT
 
 
       - name: log
         shell: bash
         run: |
           echo "workflow file: ${{ steps.current_file.outputs.currentWorkflowFile }}"
           echo "Modified: ${{ steps.set_changes.outputs.modified_keys }}"
+
+  #
+  #
+  #
+  ci-versions:
+    needs: [preflight]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Info rust
+        run: |
+          rustup show
+          cargo --version
+          cargo +nightly --version
+          cargo clippy --version
+          echo "yarn version: $(yarn --version)"
+          echo $( substrate-contracts-node --version | awk 'NF' )
+          estuary --version
+          cargo-contract --version
+
+      - name: Info forklift
+        run: forklift version
+
+      - name: Info vars
+        run: |
+          echo "COMMIT_SHA: ${{ needs.preflight.outputs.COMMIT_SHA }}"
+          echo "COMMIT_SHA_SHORT: ${{ needs.preflight.outputs.COMMIT_SHA_SHORT }}"
+          echo "SOURCE_REF_SLUG: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }}"
+          echo "REF_SLUG: ${{ needs.preflight.outputs.REF_SLUG }}"
+          echo "RUNNER: ${{ needs.preflight.outputs.RUNNER }}"
+          echo "IMAGE: ${{ needs.preflight.outputs.IMAGE }}"
+          #
           echo "github.ref: ${{ github.ref }}"
           echo "github.ref_name: ${{ github.ref_name }}"
-          echo "github.sha: ${{ github.sha }}"
+          echo "github.sha: ${{ github.sha }}"
diff --git a/.github/workflows/subsystem-benchmarks.yml b/.github/workflows/subsystem-benchmarks.yml
@@ -22,7 +22,7 @@ jobs:
   build:
     timeout-minutes: 80
     needs: [preflight]
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     strategy:

diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml
@@ -68,7 +68,7 @@ jobs:
         runners:
           [
             "${{ needs.preflight.outputs.RUNNER }}",
-            "${{ needs.preflight.outputs.OLDLINUXRUNNER }}",
+            "${{ needs.preflight.outputs.RUNNER_OLDLINUX }}",
           ]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
@@ -321,7 +321,7 @@ jobs:
   cargo-check-all-crate-macos:
     timeout-minutes: 30
     needs: [preflight]
-    runs-on: parity-macos
+    runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }}
     env:
       SKIP_WASM_BUILD: 1
     steps: