Merge branch 'master' into outbound-queue-v2

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-11-19-v202411281558"

.github/scripts/common/lib.sh

@@ -270,20 +270,19 @@ fetch_debian_package_from_s3() {
 }
 
 # Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set:
-# - RELEASE_ID
-# - GITHUB_TOKEN
-# - REPO in the form paritytech/polkadot
+# inputs: binary (polkadot), target(aarch64-apple-darwin)
 fetch_release_artifacts_from_s3() {
   BINARY=$1
-  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"}
+  TARGET=$2
+  OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${TARGET}/${BINARY}"}
   echo "OUTPUT_DIR : $OUTPUT_DIR"
 
   URL_BASE=$(get_s3_url_base $BINARY)
   echo "URL_BASE=$URL_BASE"
 
-  URL_BINARY=$URL_BASE/$VERSION/$BINARY
-  URL_SHA=$URL_BASE/$VERSION/$BINARY.sha256
-  URL_ASC=$URL_BASE/$VERSION/$BINARY.asc
+  URL_BINARY=$URL_BASE/$VERSION/$TARGET/$BINARY
+  URL_SHA=$URL_BASE/$VERSION/$TARGET/$BINARY.sha256
+  URL_ASC=$URL_BASE/$VERSION/$TARGET/$BINARY.asc
 
   # Fetch artifacts
   mkdir -p "$OUTPUT_DIR"
@@ -298,23 +297,34 @@ fetch_release_artifacts_from_s3() {
   pwd
   ls -al --color
   popd > /dev/null
-
+  unset OUTPUT_DIR
 }
 
 # Pass the name of the binary as input, it will
 # return the s3 base url
 function get_s3_url_base() {
     name=$1
     case $name in
-    polkadot | polkadot-execute-worker | polkadot-prepare-worker | staking-miner)
+      polkadot | polkadot-execute-worker | polkadot-prepare-worker )
         printf "https://releases.parity.io/polkadot"
         ;;
 
-    polkadot-parachain)
-        printf "https://releases.parity.io/cumulus"
+      polkadot-parachain)
+        printf "https://releases.parity.io/polkadot-parachain"
+        ;;
+
+      polkadot-omni-node)
+        printf "https://releases.parity.io/polkadot-omni-node"
         ;;
 
-    *)
+      chain-spec-builder)
+        printf "https://releases.parity.io/chain-spec-builder"
+        ;;
+
+      frame-omni-bencher)
+        printf "https://releases.parity.io/frame-omni-bencher"
+        ;;
+      *)
         printf "UNSUPPORTED BINARY $name"
         exit 1
         ;;
@@ -497,3 +507,16 @@ validate_stable_tag() {
         exit 1
     fi
 }
+
+# Prepare docker stable tag form the polkadot stable tag
+# input: tag (polkaodot-stableYYMM(-X) or polkadot-stableYYMM(-X)-rcX)
+# output: stableYYMM(-X) or stableYYMM(-X)-rcX
+prepare_docker_stable_tag() {
+  tag="$1"
+  if [[ "$tag" =~ stable[0-9]{4}(-[0-9]+)?(-rc[0-9]+)? ]]; then
+      echo "${BASH_REMATCH[0]}"
+  else
+      echo "Tag is invalid: $tag"
+      exit 1
+  fi
+}

.github/scripts/release/release_lib.sh

@@ -129,13 +129,69 @@ upload_s3_release() {
     echo "Working on version:  $version "
     echo "Working on platform: $target "
 
+    URL_BASE=$(get_s3_url_base $product)
+
     echo "Current content, should be empty on new uploads:"
-    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize || true
+    aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize || true
     echo "Content to be uploaded:"
-    artifacts="artifacts/$product/"
+    artifacts="release-artifacts/$target/$product/"
     ls "$artifacts"
-    aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/${product}/${version}/${target}"
+    aws s3 sync --acl public-read "$artifacts" "s3://${URL_BASE}/${version}/${target}"
     echo "Uploaded files:"
-    aws s3 ls "s3://releases.parity.io/${product}/${version}/${target}" --recursive --human-readable --summarize
-    echo "✅ The release should be at https://releases.parity.io/${product}/${version}/${target}"
+    aws s3 ls "s3://${URL_BASE}/${version}/${target}" --recursive --human-readable --summarize
+    echo "✅ The release should be at https://${URL_BASE}/${version}/${target}"
+}
+
+# Upload runtimes artifacts to s3 release bucket
+#
+# input: version (stable release tage.g. polkadot-stable2412 or polkadot-stable2412-rc1)
+# output: none
+upload_s3_runtimes_release_artifacts() {
+  alias aws='podman run --rm -it docker.io/paritytech/awscli -e AWS_ACCESS_KEY_ID -e AWS_SECRET_ACCESS_KEY -e AWS_BUCKET aws'
+
+  version=$1
+
+  echo "Working on version: $version "
+
+  echo "Current content, should be empty on new uploads:"
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize || true
+  echo "Content to be uploaded:"
+  artifacts="artifacts/runtimes/"
+  ls "$artifacts"
+  aws s3 sync --acl public-read "$artifacts" "s3://releases.parity.io/polkadot/runtimes/${version}/"
+  echo "Uploaded files:"
+  aws s3 ls "s3://releases.parity.io/polkadot/runtimes/${version}/" --recursive --human-readable --summarize
+  echo "✅ The release should be at https://releases.parity.io/polkadot/runtimes/${version}"
+}
+
+
+# Pass the name of the binary as input, it will
+# return the s3 base url
+function get_s3_url_base() {
+    name=$1
+    case $name in
+      polkadot | polkadot-execute-worker | polkadot-prepare-worker )
+        printf "releases.parity.io/polkadot"
+        ;;
+
+      polkadot-parachain)
+        printf "releases.parity.io/polkadot-parachain"
+        ;;
+
+      polkadot-omni-node)
+        printf "releases.parity.io/polkadot-omni-node"
+        ;;
+
+      chain-spec-builder)
+        printf "releases.parity.io/chain-spec-builder"
+        ;;
+
+      frame-omni-bencher)
+        printf "releases.parity.io/frame-omni-bencher"
+        ;;
+      *)
+        printf "UNSUPPORTED BINARY $name"
+        exit 1
+        ;;
+    esac
 }

.github/workflows/check-links.yml

@@ -33,7 +33,7 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.0 (22. Sep 2023)
 
       - name: Lychee link checker
-        uses: lycheeverse/lychee-action@7cd0af4c74a61395d455af97419279d86aafaede # for v1.9.1 (10. Jan 2024)
+        uses: lycheeverse/lychee-action@f81112d0d2814ded911bd23e3beaa9dda9093915 # for v1.9.1 (10. Jan 2024)
         with:
           args: >-
             --config .config/lychee.toml

.github/workflows/check-semver.yml

@@ -11,7 +11,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TOOLCHAIN: nightly-2024-10-19
+  TOOLCHAIN: nightly-2024-11-19
 
 jobs:
   preflight:
@@ -74,10 +74,15 @@ jobs:
 
       - name: install parity-publish
         # Set the target dir to cache the build.
-        run: CARGO_TARGET_DIR=./target/ cargo install [email protected].1 --locked -q
+        run: CARGO_TARGET_DIR=./target/ cargo install [email protected].2 --locked -q
 
       - name: check semver
         run: |
+          if [ -z "$PR" ]; then
+            echo "Skipping master/merge queue"
+            exit 0
+          fi
+
           export CARGO_TARGET_DIR=target
           export RUSTFLAGS='-A warnings -A missing_docs'
           export SKIP_WASM_BUILD=1

.github/workflows/checks-quick.yml

@@ -97,7 +97,6 @@ jobs:
           --exclude
           "substrate/frame/contracts/fixtures/build"
           "substrate/frame/contracts/fixtures/contracts/common"
-          "substrate/frame/revive/fixtures/build"
           "substrate/frame/revive/fixtures/contracts/common"
       - name: deny git deps
         run: python3 .github/scripts/deny-git-deps.py .

.github/workflows/command-backport.yml

@@ -29,20 +29,21 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Generate token
-        id: generate_token
-        uses: tibdex/github-app-token@v2.1.0
+      - name: Generate content write token for the release automation
+        id: generate_write_token
+        uses: actions/create-github-app-token@v1
         with:
-          app_id: ${{ secrets.CMD_BOT_APP_ID }}
-          private_key: ${{ secrets.CMD_BOT_APP_KEY }}
+            app-id: ${{ vars.RELEASE_AUTOMATION_APP_ID }}
+            private-key: ${{ secrets.RELEASE_AUTOMATION_APP_PRIVATE_KEY }}
+            owner: paritytech
 
       - name: Create backport pull requests
         uses: korthout/backport-action@v3
         id: backport
         with:
           target_branches: stable2407 stable2409 stable2412
           merge_commits: skip
-          github_token: ${{ steps.generate_token.outputs.token }}
+          github_token: ${{ steps.generate_write_token.outputs.token }}
           pull_description: |
             Backport #${pull_number} into `${target_branch}` from ${pull_author}.
 
@@ -86,7 +87,7 @@ jobs:
             const reviewer = '${{ github.event.pull_request.user.login }}';
 
             for (const pullNumber of pullNumbers) {
-              await github.pulls.createReviewRequest({
+              await github.pulls.requestReviewers({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
                 pull_number: parseInt(pullNumber),

.github/workflows/publish-check-compile.yml

@@ -0,0 +1,48 @@
+name: Check publish build
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  merge_group:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  preflight:
+    uses: ./.github/workflows/reusable-preflight.yml
+
+  check-publish:
+    timeout-minutes: 90
+    needs: [preflight]
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    container:
+      image: ${{ needs.preflight.outputs.IMAGE }}
+    steps:
+      - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
+
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@82a92a6e8fbeee089604da2575dc567ae9ddeaab # v2.7.5
+        with:
+          cache-on-failure: true
+
+      - name: install parity-publish
+        run: cargo install [email protected] --locked -q
+
+      - name: parity-publish update plan
+        run: parity-publish --color always plan --skip-check --prdoc prdoc/
+
+      - name: parity-publish apply plan
+        run: parity-publish --color always apply --registry
+
+      - name: parity-publish check compile
+        run: |
+          packages="$(parity-publish apply --print)"
+
+          if [ -n "$packages" ]; then
+            cargo --color always check $(printf -- '-p %s ' $packages)
+          fi

.github/workflows/publish-check-crates.yml

@@ -24,7 +24,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.8.0 --locked -q
+        run: cargo install parity-publish@0.10.2 --locked -q
 
       - name: parity-publish check
         run: parity-publish --color always check --allow-unpublished

.github/workflows/publish-claim-crates.yml

@@ -18,7 +18,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.8.0 --locked -q
+        run: cargo install parity-publish@0.10.2 --locked -q
 
       - name: parity-publish claim
         env: