Merge branch 'master' into mrcnski/pvf-test-instructions

paritytech · Nov 26, 2023 · 8f8f7b3 · 8f8f7b3
2 parents 95b869c + cfa19c3
commit 8f8f7b3
Show file tree

Hide file tree

Showing 202 changed files with 3,754 additions and 3,203 deletions.
diff --git a/.github/review-bot.yml b/.github/review-bot.yml
@@ -9,7 +9,7 @@ rules:
         - ^\.config/nextest.toml
         - ^\.cargo/.*
       exclude: 
-        - ^./gitlab/pipeline/zombienet.*
+        - ^\.gitlab/pipeline/zombienet.*
     minApprovals: 2
     type: basic
     teams:

diff --git a/.github/workflows/check-publish.yml b/.github/workflows/check-publish.yml
@@ -10,10 +10,7 @@ on:
 
 jobs:
   check-publish:
-    strategy:
-      matrix:
-        os: ["ubuntu-latest"]
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0
 
@@ -23,7 +20,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish --profile dev
+        run: cargo install parity-publish@0.3.0
 
       - name: parity-publish check
         run: parity-publish check --allow-unpublished
diff --git a/.github/workflows/claim-crates.yml b/.github/workflows/claim-crates.yml
@@ -18,7 +18,7 @@ jobs:
           cache-on-failure: true
 
       - name: install parity-publish
-        run: cargo install parity-publish@0.2.0
+        run: cargo install parity-publish@0.3.0
 
       - name: parity-publish claim
         env:

diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml
@@ -0,0 +1,24 @@
+name: Merge-Queue
+
+on:
+  merge_group:
+
+jobs:
+  trigger-merge-queue-action:
+    runs-on: ubuntu-latest
+    environment: master
+    steps:
+      - name: Generate token
+        id: app_token
+        uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0
+        with:
+          app_id: ${{ secrets.REVIEW_APP_ID }}
+          private_key: ${{ secrets.REVIEW_APP_KEY }}
+      - name: Add Merge Queue status check
+        uses: billyjbryant/create-status-check@3e6fa0ac599d10d9588cf9516ca4330ef669b858 # v2
+        with:
+          authToken: ${{ steps.app_token.outputs.token }}
+          context: 'review-bot'
+          description: 'PRs for merge queue gets approved'
+          state: 'success'
+          sha: ${{ github.event.merge_group.head_commit.id }}
diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml
@@ -268,7 +268,7 @@ jobs:
 
       - name: Build and push
         id:   docker_build
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 #  v5.0.0
+        uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 #  v5.1.0
         with:
           push: true
           file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -30,8 +30,7 @@ variables:
   RUSTY_CACHIER_COMPRESSION_METHOD: zstd
   NEXTEST_FAILURE_OUTPUT: immediate-final
   NEXTEST_SUCCESS_OUTPUT: final
-  ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.80"
-  DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}"
+  DOCKER_IMAGES_VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}"
 
 default:
   retry:
@@ -199,9 +198,6 @@ default:
     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
     - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues
 
-.zombienet-refs:
-  extends: .build-refs
-
 include:
   # check jobs
   - .gitlab/pipeline/check.yml

diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml
@@ -101,16 +101,16 @@ check-rust-feature-propagation:
       export RUST_LOG=remote-ext=debug,runtime=debug
 
       echo "---------- Downloading try-runtime CLI ----------"
-      curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.3.3/try-runtime-x86_64-unknown-linux-musl -o try-runtime
+      curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.0/try-runtime-x86_64-unknown-linux-musl -o try-runtime
       chmod +x ./try-runtime
 
       echo "---------- Building ${PACKAGE} runtime ----------"
       time cargo build --release --locked -p "$PACKAGE" --features try-runtime
 
       echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------"
-      time ./try-runtime \
+      time ./try-runtime ${COMMAND_EXTRA_ARGS} \
           --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \
-          on-runtime-upgrade --checks=pre-and-post ${EXTRA_ARGS} live --uri ${URI}
+          on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI}
 
 # Check runtime migrations for Parity managed relay chains
 check-runtime-migration-westend:
@@ -124,7 +124,7 @@ check-runtime-migration-westend:
     PACKAGE: "westend-runtime"
     WASM: "westend_runtime.compact.compressed.wasm"
     URI: "wss://westend-try-runtime-node.parity-chains.parity.io:443"
-    EXTRA_ARGS: "--no-weight-warnings"
+    SUBCOMMAND_EXTRA_ARGS: "--no-weight-warnings"
 
 check-runtime-migration-rococo:
   stage: check
@@ -137,7 +137,7 @@ check-runtime-migration-rococo:
     PACKAGE: "rococo-runtime"
     WASM: "rococo_runtime.compact.compressed.wasm"
     URI: "wss://rococo-try-runtime-node.parity-chains.parity.io:443"
-    EXTRA_ARGS: "--no-weight-warnings"
+    SUBCOMMAND_EXTRA_ARGS: "--no-weight-warnings"
 
 # Check runtime migrations for Parity managed asset hub chains
 check-runtime-migration-asset-hub-westend:
@@ -151,7 +151,7 @@ check-runtime-migration-asset-hub-westend:
     PACKAGE: "asset-hub-westend-runtime"
     WASM: "asset_hub_westend_runtime.compact.compressed.wasm"
     URI: "wss://westend-asset-hub-rpc.polkadot.io:443"
-    
+
 check-runtime-migration-asset-hub-rococo:
   stage: check
   extends:
@@ -214,6 +214,7 @@ check-runtime-migration-collectives-westend:
     PACKAGE: "collectives-westend-runtime"
     WASM: "collectives_westend_runtime.compact.compressed.wasm"
     URI: "wss://westend-collectives-rpc.polkadot.io:443"
+    COMMAND_EXTRA_ARGS: "--disable-spec-name-check"
 
 find-fail-ci-phrase:
   stage: check

diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml
@@ -63,16 +63,16 @@ publish-rustdoc:
   after_script:
     - rm -rf .git/ ./*
 
-# cumulus
-
+# note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE
   variables:
     DOCKERFILE: "" # docker/path-to.Dockerfile
     IMAGE_NAME: "" # docker.io/paritypr/image_name
   script:
-    # Exit if the job is not running in a merge queue
-    # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi
+    # Dockertag should differ in a merge queue
+    # TODO: test this
+    # - if [[ $CI_COMMIT_REF_NAME == *"gh-readonly-queue"* ]]; export DOCKER_IMAGES_VERSION="${CI_COMMIT_SHORT_SHA}"; fi
     - $BUILDAH_COMMAND build
       --format=docker
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
@@ -112,59 +112,6 @@ build-push-image-test-parachain:
   variables:
     DOCKERFILE: "docker/dockerfiles/test-parachain_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/test-parachain"
-# publish-s3:
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .publish-refs
-#   image: paritytech/awscli:latest
-#   needs:
-#     - job: build-linux-stable-cumulus
-#       artifacts: true
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/${ARCH}-${DOCKER_OS}"
-#   script:
-#     - echo "___Publishing a binary with debug assertions!___"
-#     - echo "___VERSION = $(cat ./artifacts/VERSION) ___"
-#     - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/
-#     - echo "___Updating objects in latest path___"
-#     - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/
-#   after_script:
-#     - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
-#       --recursive --human-readable --summarize
-
-# publish-benchmarks-assets-s3: &publish-benchmarks
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .benchmarks-refs
-#   image: paritytech/awscli:latest
-#   needs:
-#     - job: benchmarks-assets
-#       artifacts: true
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-assets"
-#   script:
-#     - echo "___Publishing benchmark results___"
-#     - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/
-#   after_script:
-#     - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize
-
-# publish-benchmarks-collectives-s3:
-#   <<: *publish-benchmarks
-#   variables:
-#     GIT_STRATEGY: none
-#     BUCKET: "releases.parity.io"
-#     PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-collectives"
-#   needs:
-#     - job: benchmarks-collectives
-#       artifacts: true
-
-### Polkadot
 
 build-push-image-polkadot-debug:
   stage: publish
@@ -217,143 +164,3 @@ build-push-image-substrate-pr:
   variables:
     DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/substrate"
-# old way
-
-# .build-push-image-polkadot:
-#   before_script:
-#     # - test -s ./artifacts/VERSION || exit 1
-#     # - test -s ./artifacts/EXTRATAG || exit 1
-#     - VERSION="$(cat ./artifacts/VERSION)"
-#     - EXTRATAG="$(cat ./artifacts/EXTRATAG)"
-#     - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})"
-#   script:
-#     # - test "$DOCKER_USER" -a "$DOCKER_PASS" ||
-#     #   ( echo "no docker credentials provided"; exit 1 )
-#     - cd ./artifacts
-#     - $BUILDAH_COMMAND build
-#       --format=docker
-#       --build-arg VCS_REF="${CI_COMMIT_SHA}"
-#       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
-#       --build-arg IMAGE_NAME="${IMAGE_NAME}"
-#       --tag "$IMAGE_NAME:$VERSION"
-#       --tag "$IMAGE_NAME:$EXTRATAG"
-#       --file ${DOCKERFILE} .
-#     # The job will success only on the protected branch
-#     # - echo "$DOCKER_PASS" |
-#     #   buildah login --username "$DOCKER_USER" --password-stdin docker.io
-#     # - $BUILDAH_COMMAND info
-#     # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION"
-#     # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$EXTRATAG"
-#   after_script:
-#     - buildah logout --all
-
-# publish-polkadot-debug-image:
-#   stage: publish
-#   image: ${BUILDAH_IMAGE}
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#   rules:
-#     - if: $CI_PIPELINE_SOURCE == "web"
-#     - if: $CI_PIPELINE_SOURCE == "schedule"
-#     - if: $CI_COMMIT_REF_NAME == "master"
-#     - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
-#     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1
-#   variables:
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile
-#     DOCKERFILE: polkadot_injected_debug.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/polkadot-debug
-#   needs:
-#     - job: build-linux-stable
-#       artifacts: true
-#   after_script:
-#     # pass artifacts to the zombienet-tests job
-#     # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
-#     - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env
-#     - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/parachains.env
-#     expire_in: 1 days
-
-# publish-test-collators-image:
-#   # service image for zombienet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/collator_injected.Dockerfile
-#     DOCKERFILE: collator_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/colander
-#   needs:
-#     - job: build-test-collators
-#       artifacts: true
-#   after_script:
-#     - buildah logout --all
-#     # pass artifacts to the zombienet-tests job
-#     - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env
-#     - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/collator.env
-
-# publish-malus-image:
-#   # service image for Simnet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     # docker/dockerfiles/malus_injected.Dockerfile
-#     DOCKERFILE: malus_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/malus
-#   needs:
-#     - job: build-malus
-#       artifacts: true
-#   after_script:
-#     - buildah logout "$IMAGE_NAME"
-#     # pass artifacts to the zombienet-tests job
-#     - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env
-#     - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env
-#   artifacts:
-#     reports:
-#       # this artifact is used in zombienet-tests job
-#       dotenv: ./artifacts/malus.env
-
-# substrate
-
-# publish-substrate-image-pr:
-#   # service image for zombienet
-#   stage: publish
-#   extends:
-#     - .kubernetes-env
-#     - .build-push-image-polkadot
-#     - .zombienet-refs
-#   variables:
-#     CI_IMAGE: ${BUILDAH_IMAGE}
-#     GIT_STRATEGY: none
-#     DOCKER_USER: ${PARITYPR_USER}
-#     DOCKER_PASS: ${PARITYPR_PASS}
-#     DOCKERFILE: substrate_injected.Dockerfile
-#     IMAGE_NAME: docker.io/paritypr/substrate
-#   needs:
-#     - job: build-linux-substrate
-#       artifacts: true
-#   after_script:
-#     - buildah logout "$IMAGE_NAME"
diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml
@@ -29,7 +29,7 @@ test-linux-stable:
         --locked \
         --release \
         --no-fail-fast \
-        --features try-runtime,experimental \
+        --features try-runtime,experimental,ci-only-tests \
         --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
     # Upload tests results to Elasticsearch
     - echo "Upload test results to Elasticsearch"
@@ -522,4 +522,4 @@ test-syscalls:
     - if [[ "$CI_JOB_STATUS" == "failed" ]]; then
         printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n";
       fi
-  allow_failure: true # TODO: remove this once we have an idea how often the syscall lists will change
+  allow_failure: false # this rarely triggers in practice
diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml
@@ -1,3 +1,8 @@
+.zombienet-refs:
+  extends: .build-refs
+  variables:
+    ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.83"
+
 include:
   # substrate tests
   - .gitlab/pipeline/zombienet/substrate.yml