Merge remote-tracking branch 'origin/master' into na-fix-rpc-storage-…

…iter
paritytech · Sep 24, 2024 · dc17cc4 · dc17cc4
2 parents 05386e8 + c987da3
commit dc17cc4
Show file tree

Hide file tree

Showing 163 changed files with 4,951 additions and 2,109 deletions.
diff --git a/.github/workflows/check-licenses.yml b/.github/workflows/check-licenses.yml
@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
-      - uses: actions/[email protected].3
+      - uses: actions/[email protected].4
         with:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"
@@ -56,7 +56,7 @@ jobs:
     steps:
       - name: Checkout sources
         uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
-      - uses: actions/[email protected].3
+      - uses: actions/[email protected].4
         with:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"

diff --git a/.github/workflows/checks-quick.yml b/.github/workflows/checks-quick.yml
@@ -102,7 +102,7 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
       - name: Setup Node.js
-        uses: actions/[email protected].3
+        uses: actions/[email protected].4
         with:
           node-version: "18.x"
           registry-url: "https://npm.pkg.github.com"

diff --git a/.github/workflows/misc-sync-templates.yml b/.github/workflows/misc-sync-templates.yml
@@ -157,7 +157,7 @@ jobs:
         timeout-minutes: 90
       - name: Create PR on failure
         if: failure() && steps.check-compilation.outcome == 'failure'
-        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 # v5
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v5
         with:
           path: "${{ env.template-path }}"
           token: ${{ steps.app_token.outputs.token }}
@@ -167,7 +167,7 @@ jobs:
           body: "The template has NOT been successfully built and needs to be inspected."
           branch: "update-template/${{ github.event.inputs.stable_release_branch }}"
       - name: Create PR on success
-        uses: peter-evans/create-pull-request@8867c4aba1b742c39f8d0ba35429c2dfa4b6cb20 # v5
+        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v5
         with:
           path: "${{ env.template-path }}"
           token: ${{ steps.app_token.outputs.token }}

diff --git a/.github/workflows/reusable-preflight.yml b/.github/workflows/reusable-preflight.yml
@@ -26,36 +26,77 @@ on:
       IMAGE:
         value: ${{ jobs.preflight.outputs.IMAGE }}
         description: "CI image"
+
+      # Runners
+      # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners
       RUNNER:
         value: ${{ jobs.preflight.outputs.RUNNER }}
         description: |
-          Runner name.
+          Main runner for resource-intensive tasks
+          By default we use spot machines that can be terminated at any time.
+          Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
+      RUNNER_OLDLINUX:
+        value: ${{ jobs.preflight.outputs.RUNNER_OLDLINUX }}
+        description: |
+          parity-oldlinux
           By default we use spot machines that can be terminated at any time.
           Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
-      OLDLINUXRUNNER:
-        value: ${{ jobs.preflight.outputs.OLDLINUXRUNNER }}
+      RUNNER_DEFAULT:
+        value: ${{ jobs.preflight.outputs.RUNNER_DEFAULT }}
+        description: "Relatively lightweight runner. When `ubuntu-latest` is not enough"
+      RUNNER_WEIGHTS:
+        value: ${{ jobs.preflight.outputs.RUNNER_WEIGHTS }}
+      RUNNER_BENCHMARK:
+        value: ${{ jobs.preflight.outputs.RUNNER_BENCHMARK }}
+      RUNNER_MACOS:
+        value: ${{ jobs.preflight.outputs.RUNNER_MACOS }}
+
+      # Vars
+      SOURCE_REF_SLUG:
+        value: ${{ jobs.preflight.outputs.SOURCE_REF_SLUG }}
+        description: "Name of the current branch for `push` or source branch for `pull_request` with `/` replaced by `_`. Does not exists in merge_group"
+      REF_SLUG:
+        value: ${{ jobs.preflight.outputs.REF_SLUG }}
+        description: |
+          Name of the current revision (depending on the event) with `/` replaced by `_`, e.g:
+          push - master
+          pull_request - 49_merge
+          merge_group - gh-readonly-queue_master_pr-49-38d43798a986430231c828b2c762997f818ac012
 
-      SOURCE_REF_NAME:
-        value: ${{ jobs.preflight.outputs.SOURCE_REF_NAME }}
-        description: "Name of the current branch for `push` or source branch for `pull_request`"
       COMMIT_SHA:
         value: ${{ jobs.preflight.outputs.COMMIT_SHA }}
-        description: "Sha of the current commit for `push` or head of the source branch for `pull_request`"
+        description: "Sha of the current revision"
+      COMMIT_SHA_SHORT:
+        value: ${{ jobs.preflight.outputs.COMMIT_SHA_SHORT }}
+        description: "Sha of the current revision, 8-symbols long"
 
 jobs:
+
+  #
+  #
+  #
   preflight:
     runs-on: ubuntu-latest
     outputs:
       changes_rust: ${{ steps.set_changes.outputs.rust_any_changed || steps.set_changes.outputs.currentWorkflow_any_changed }}
       changes_currentWorkflow: ${{ steps.set_changes.outputs.currentWorkflow_any_changed }}
 
       IMAGE: ${{ steps.set_image.outputs.IMAGE }}
+
+      # Runners
+      # https://github.com/paritytech/ci_cd/wiki/GitHub#paritytech-self-hosted-runners
       RUNNER: ${{ steps.set_runner.outputs.RUNNER }}
-      OLDLINUXRUNNER: ${{ steps.set_runner.outputs.OLDLINUXRUNNER }}
+      RUNNER_OLDLINUX: ${{ steps.set_runner.outputs.RUNNER_OLDLINUX }}
+      RUNNER_DEFAULT: ${{ steps.set_runner.outputs.RUNNER_DEFAULT }}
+      RUNNER_WEIGHTS: ${{ steps.set_runner.outputs.RUNNER_WEIGHTS }}
+      RUNNER_BENCHMARK: ${{ steps.set_runner.outputs.RUNNER_BENCHMARK }}
+      RUNNER_MACOS: ${{ steps.set_runner.outputs.RUNNER_MACOS }}
 
-      SOURCE_REF_NAME: ${{ steps.set_vars.outputs.SOURCE_REF_NAME }}
-      COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }}
+      SOURCE_REF_SLUG: ${{ steps.set_vars.outputs.SOURCE_REF_SLUG }}
+      REF_SLUG: ${{ steps.set_vars.outputs.REF_SLUG }}
 
+      COMMIT_SHA: ${{ steps.set_vars.outputs.COMMIT_SHA }}
+      COMMIT_SHA_SHORT: ${{ steps.set_vars.outputs.COMMIT_SHA_SHORT }}
 
     steps:
 
@@ -64,7 +105,8 @@ jobs:
       #
       # Set changes
       #
-      - id: current_file
+      - name: Current file
+        id: current_file
         shell: bash
         run: |
           echo "currentWorkflowFile=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT
@@ -98,34 +140,82 @@ jobs:
       # By default we use spot machines that can be terminated at any time.
       # Merge queues use persistent runners to avoid kicking off from queue when the runner is terminated.
       #
-      - id: set_runner
+      - name: Set runner
+        id: set_runner
         shell: bash
         run: |
+          echo "RUNNER_DEFAULT=parity-default" >> $GITHUB_OUTPUT
+          echo "RUNNER_WEIGHTS=parity-weights" >> $GITHUB_OUTPUT
+          echo "RUNNER_BENCHMARK=parity-benchmark" >> $GITHUB_OUTPUT
+          echo "RUNNER_MACOS=parity-macos" >> $GITHUB_OUTPUT
+          #
           # Run merge queues on persistent runners
           if [[ $GITHUB_REF_NAME == *"gh-readonly-queue"* ]]; then
-            echo "RUNNER=arc-runners-polkadot-sdk-beefy-persistent" >> $GITHUB_OUTPUT
-            echo "OLDLINUXRUNNER=oldlinux-persistent" >> $GITHUB_OUTPUT
+            echo "RUNNER=parity-large-persistent" >> $GITHUB_OUTPUT
+            echo "RUNNER_OLDLINUX=parity-oldlinux-persistent" >> $GITHUB_OUTPUT
           else
-            echo "RUNNER=arc-runners-polkadot-sdk-beefy" >> $GITHUB_OUTPUT
-            echo "OLDLINUXRUNNER=oldlinux" >> $GITHUB_OUTPUT
+            echo "RUNNER=parity-large" >> $GITHUB_OUTPUT
+            echo "RUNNER_OLDLINUX=parity-oldlinux" >> $GITHUB_OUTPUT
           fi
 
       #
       # Set vars
       #
-      - id: set_vars
+      - name: Set vars
+        id: set_vars
         shell: bash
         run: |
-          export BRANCH_NAME=${{ github.head_ref || github.ref_name }}
-          echo "SOURCE_REF_NAME=${BRANCH_NAME//\//-}" >> $GITHUB_OUTPUT
-          echo "COMMIT_SHA=${{ github.event.pull_request.head.sha || github.sha }}" >> $GITHUB_OUTPUT
+          export SOURCE_REF_NAME=${{ github.head_ref || github.ref_name }}
+          echo "SOURCE_REF_SLUG=${SOURCE_REF_NAME//\//_}" >> $GITHUB_OUTPUT
+          #
+          export COMMIT_SHA=${{ github.sha }}
+          echo "COMMIT_SHA=$COMMIT_SHA" >> $GITHUB_OUTPUT
+          echo "COMMIT_SHA_SHORT=${COMMIT_SHA:0:8}" >> $GITHUB_OUTPUT
+          #
+          export REF_NAME=${{ github.ref_name }}
+          echo "REF_SLUG=${REF_NAME//\//_}" >> $GITHUB_OUTPUT
 
 
       - name: log
         shell: bash
         run: |
           echo "workflow file: ${{ steps.current_file.outputs.currentWorkflowFile }}"
           echo "Modified: ${{ steps.set_changes.outputs.modified_keys }}"
+
+  #
+  #
+  #
+  ci-versions:
+    needs: [preflight]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Info rust
+        run: |
+          rustup show
+          cargo --version
+          cargo +nightly --version
+          cargo clippy --version
+          echo "yarn version: $(yarn --version)"
+          echo $( substrate-contracts-node --version | awk 'NF' )
+          estuary --version
+          cargo-contract --version
+
+      - name: Info forklift
+        run: forklift version
+
+      - name: Info vars
+        run: |
+          echo "COMMIT_SHA: ${{ needs.preflight.outputs.COMMIT_SHA }}"
+          echo "COMMIT_SHA_SHORT: ${{ needs.preflight.outputs.COMMIT_SHA_SHORT }}"
+          echo "SOURCE_REF_SLUG: ${{ needs.preflight.outputs.SOURCE_REF_SLUG }}"
+          echo "REF_SLUG: ${{ needs.preflight.outputs.REF_SLUG }}"
+          echo "RUNNER: ${{ needs.preflight.outputs.RUNNER }}"
+          echo "IMAGE: ${{ needs.preflight.outputs.IMAGE }}"
+          #
           echo "github.ref: ${{ github.ref }}"
           echo "github.ref_name: ${{ github.ref_name }}"
-          echo "github.sha: ${{ github.sha }}"
+          echo "github.sha: ${{ github.sha }}"
diff --git a/.github/workflows/subsystem-benchmarks.yml b/.github/workflows/subsystem-benchmarks.yml
@@ -22,7 +22,7 @@ jobs:
   build:
     timeout-minutes: 80
     needs: [preflight]
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    runs-on: ${{ needs.preflight.outputs.RUNNER_BENCHMARK }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     strategy:

diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml
@@ -68,7 +68,7 @@ jobs:
         runners:
           [
             "${{ needs.preflight.outputs.RUNNER }}",
-            "${{ needs.preflight.outputs.OLDLINUXRUNNER }}",
+            "${{ needs.preflight.outputs.RUNNER_OLDLINUX }}",
           ]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

diff --git a/.github/workflows/tests-misc.yml b/.github/workflows/tests-misc.yml
@@ -321,7 +321,7 @@ jobs:
   cargo-check-all-crate-macos:
     timeout-minutes: 30
     needs: [preflight]
-    runs-on: parity-macos
+    runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }}
     env:
       SKIP_WASM_BUILD: 1
     steps:
@@ -334,7 +334,7 @@ jobs:
       - name: Set up Homebrew
         uses: Homebrew/actions/setup-homebrew@1ccc07ccd54b6048295516a3eb89b192c35057dc # master from 12.09.2024
       - name: Install rust ${{ env.RUST_VERSION }}
-        uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
+        uses: actions-rust-lang/setup-rust-toolchain@4d1965c9142484e48d40c19de54b5cba84953a06 # v1.10.0
         with:
           cache: false
           toolchain: ${{ env.RUST_VERSION }}

diff --git a/.gitlab/pipeline/short-benchmarks.yml b/.gitlab/pipeline/short-benchmarks.yml
@@ -26,6 +26,27 @@ short-benchmark-westend: &short-bench
   script:
     - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
 
+short-benchmark-rococo: &short-bench
+  stage: short-benchmarks
+  extends:
+    - .docker-env
+    - .common-refs
+  needs:
+    - job: build-short-benchmark
+      artifacts: true
+  variables:
+    RUNTIME: rococo
+    # Enable debug assertions since we are running optimized builds for testing
+    # but still want to have debug assertions.
+    RUSTFLAGS: "-C debug-assertions -D warnings"
+    RUST_BACKTRACE: "full"
+    WASM_BUILD_NO_COLOR: 1
+    WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
+  tags:
+    - benchmark
+  script:
+    - ./artifacts/polkadot benchmark pallet --chain $RUNTIME-dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1
+
 # run short-benchmarks for system parachain runtimes from cumulus
 
 .short-benchmark-cumulus: &short-bench-cumulus