Merge branch 'main' into oh_cancel_harness_branch

.dockerignore

@@ -24,6 +24,7 @@ keys/setup
 !Cargo.toml
 !contracts/
 !setup_2\^26.key
+!setup_2\^24.key
 # It's required to remove .git from contracts,
 # otherwise yarn tries to use .git parent directory that
 # doesn't exist.
@@ -45,3 +46,4 @@ contracts/.git
 !etc/env/dev.toml
 !etc/env/consensus_secrets.yaml
 !etc/env/consensus_config.yaml
+!rust-toolchain

.github/pull_request_template.md

@@ -18,4 +18,3 @@
 - [ ] Tests for the changes have been added / updated.
 - [ ] Documentation comments have been added / updated.
 - [ ] Code has been formatted via `zk fmt` and `zk lint`.
-- [ ] Spellcheck has been run via `zk spellcheck`.

.github/release-please/manifest.json

@@ -1,4 +1,4 @@
 {
-  "core": "24.1.0",
-  "prover": "14.0.0"
+  "core": "24.8.0",
+  "prover": "15.0.0"
 }

.github/workflows/build-contract-verifier-template.yml

@@ -31,13 +31,14 @@ jobs:
     runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
     strategy:
       matrix:
-          components:
-            - contract-verifier
-          platforms:
-            - linux/amd64
+        components:
+          - contract-verifier
+          - verified-sources-fetcher
+        platforms:
+          - linux/amd64
 
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           submodules: "recursive"
 
@@ -56,7 +57,7 @@ jobs:
           filtered_tag=""
           while [ true ]; do
             echo "Page: $page"
-            tags=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \
+            tags=$(run_retried curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \
               "https://api.github.com/repos/matter-labs/era-contracts/tags?per_page=100&page=${page}" | jq .)
             if [ $(jq length <<<"$tags") -eq 0 ]; then
               echo "No tag found on all pages."
@@ -72,9 +73,9 @@ jobs:
           done
           echo "Contracts tag is: ${filtered_tag}"
           mkdir -p ./contracts
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz
           tar -C ./contracts -zxf l1-contracts.tar.gz
           tar -C ./contracts -zxf l2-contracts.tar.gz
           tar -C ./contracts -zxf system-contracts.tar.gz
@@ -101,7 +102,9 @@ jobs:
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
           mkdir -p ./volumes/postgres
+          run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
+          ci_run pre_download_compilers.sh
           ci_run sccache --start-server
 
       - name: init
@@ -112,7 +115,7 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
           ci_run zk || true
           ci_run yarn zk build
-          ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
+          ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
       - name: build contracts
         if: env.BUILD_CONTRACTS == 'true'
@@ -135,14 +138,14 @@ jobs:
           COMPONENT: ${{ matrix.components }}
           PLATFORM: ${{ matrix.platforms }}
         run: |
-          ci_run rustup default nightly-2023-08-21
+          ci_run run_retried rustup default nightly-2024-05-07
           platform=$(echo $PLATFORM | tr '/' '-')
           ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT
       - name: Show sccache stats
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   create_manifest:
     name: Create release manifest
@@ -154,10 +157,12 @@ jobs:
         component:
           - name: contract-verifier
             platform: linux/amd64
+          - name: verified-sources-fetcher
+            platform: linux/amd64
     env:
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
 
       - name: login to Docker registries
         run: |

.github/workflows/build-core-template.yml

@@ -36,18 +36,18 @@ jobs:
     runs-on: ${{ fromJSON('["matterlabs-ci-runner", "matterlabs-ci-runner-arm"]')[contains(matrix.platforms, 'arm')] }}
     strategy:
       matrix:
-          components:
-            - server-v2
-            - external-node
-            - snapshots-creator
-          platforms:
-            - linux/amd64
-          include:
-            - components: external-node
-              platforms: linux/arm64
+        components:
+          - server-v2
+          - external-node
+          - snapshots-creator
+        platforms:
+          - linux/amd64
+        include:
+          - components: external-node
+            platforms: linux/arm64
 
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           submodules: "recursive"
 
@@ -66,7 +66,7 @@ jobs:
           filtered_tag=""
           while [ true ]; do
             echo "Page: $page"
-            tags=$(curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \
+            tags=$(run_retried curl -s -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" -H "Accept: application/vnd.github+json" \
               "https://api.github.com/repos/matter-labs/era-contracts/tags?per_page=100&page=${page}" | jq .)
             if [ $(jq length <<<"$tags") -eq 0 ]; then
               echo "No tag found on all pages."
@@ -82,9 +82,9 @@ jobs:
           done
           echo "Contracts tag is: ${filtered_tag}"
           mkdir -p ./contracts
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz
-          curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l1-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/l2-contracts.tar.gz
+          run_retried curl -s -LO https://github.com/matter-labs/era-contracts/releases/download/${filtered_tag}/system-contracts.tar.gz
           tar -C ./contracts -zxf l1-contracts.tar.gz
           tar -C ./contracts -zxf l2-contracts.tar.gz
           tar -C ./contracts -zxf system-contracts.tar.gz
@@ -111,7 +111,9 @@ jobs:
         run: |
           echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
           mkdir -p ./volumes/postgres
+          run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
+          ci_run pre_download_compilers.sh
           ci_run sccache --start-server
 
       - name: init
@@ -122,7 +124,7 @@ jobs:
           ci_run git config --global --add safe.directory /usr/src/zksync/contracts
           ci_run zk || true
           ci_run yarn zk build
-          ci_run curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
+          ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^26.key
 
       - name: build contracts
         if: env.BUILD_CONTRACTS == 'true'
@@ -145,14 +147,14 @@ jobs:
           COMPONENT: ${{ matrix.components }}
           PLATFORM: ${{ matrix.platforms }}
         run: |
-          ci_run rustup default nightly-2023-08-21
+          ci_run run_retried rustup default nightly-2024-05-07
           platform=$(echo $PLATFORM | tr '/' '-')
           ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT
       - name: Show sccache stats
         if: always()
         run: |
-          ci_run sccache --show-stats
-          ci_run cat /tmp/sccache_log.txt
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true
 
   create_manifest:
     name: Create release manifest
@@ -168,10 +170,11 @@ jobs:
             platform: linux/amd64,linux/arm64
           - name: snapshots-creator
             platform: linux/amd64
+
     env:
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}${{ (inputs.en_alpha_release && matrix.component.name == 'external-node') && '-alpha' || '' }}
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
 
       - name: login to Docker registries
         run: |

.github/workflows/build-docker-from-tag.yml

@@ -29,7 +29,7 @@ jobs:
       prover_fri_cpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.cpu_short_commit_sha }}
       prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
       - name: Generate output with git tag
         id: set
         run: |
@@ -92,3 +92,4 @@ jobs:
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
+      protocol_version: ${{ needs.build-push-prover-images.outputs.protocol_version }}

.github/workflows/build-local-node-docker.yml

@@ -18,7 +18,7 @@ jobs:
     name: Local Node - Build and Push Docker Image
     runs-on: [matterlabs-ci-runner]
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           submodules: "recursive"
 
@@ -51,6 +51,7 @@ jobs:
       - name: start-services
         run: |
           mkdir -p ./volumes/postgres
+          run_retried docker compose pull zk postgres
           docker compose up -d zk postgres
 
       - name: init

.github/workflows/build-prover-fri-gpu-gar.yml

@@ -11,13 +11,17 @@ on:
         description: "Commit sha for downloading setup data from bucket dir"
         required: true
         type: string
+      protocol_version:
+        description: "Protocol version to be included in the images tag"
+        required: true
+        type: string
 
 jobs:
   build-gar-prover-fri-gpu:
     name: Build prover FRI GPU GAR
     runs-on: [matterlabs-ci-runner]
     steps:
-      - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
           submodules: "recursive"
 
@@ -44,6 +48,7 @@ jobs:
           push: true
           tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login to Asia GAR
         run: |
@@ -54,6 +59,9 @@ jobs:
           docker buildx imagetools create \
             --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
       - name: Login to Europe GAR
         run: |
@@ -64,3 +72,6 @@ jobs:
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.image_tag_suffix }}
+          docker buildx imagetools create \
+            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}