Merge branch 'main' into dz-verify-inclusion-flag

.github/release-please/manifest.json

@@ -1,5 +1,5 @@
 {
-  "core": "24.16.0",
-  "prover": "16.3.0",
-  "zk_toolbox": "0.1.1"
+  "core": "24.21.0",
+  "prover": "16.4.0",
+  "zk_toolbox": "0.1.2"
 }

.github/workflows/build-docker-from-tag.yml

@@ -94,6 +94,20 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
+  build-push-witness-generator-image-avx512:
+    name: Build and push image
+    needs: [setup]
+    uses: ./.github/workflows/build-witness-generator-template.yml
+    if: contains(github.ref_name, 'prover')
+    with:
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
+      ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
+      CUDA_ARCH: "60;70;75;89"
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
   build-gar-prover-fri-gpu:
     name: Build GAR prover FRI GPU
     needs: [setup, build-push-prover-images]

.github/workflows/build-prover-template.yml

@@ -53,6 +53,7 @@ jobs:
           - prover-gpu-fri
           - witness-vector-generator
           - prover-fri-gateway
+          - prover-job-monitor
           - proof-fri-gpu-compressor
     outputs:
       protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
@@ -163,7 +164,7 @@ jobs:
         run: |
           ci_run sccache --show-stats || true
           ci_run cat /tmp/sccache_log.txt || true
-  
+
   copy-images:
     name: Copy images between docker registries
     needs: build-images
@@ -197,4 +198,3 @@ jobs:
           docker buildx imagetools create \
             --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/${{ matrix.component }}:2.0-${{ needs.build-images.outputs.protocol_version }}-${{ inputs.image_tag_suffix }}
-          

.github/workflows/build-witness-generator-template.yml

@@ -0,0 +1,172 @@
+name: Build witness generator image with custom compiler flags
+on:
+  workflow_call:
+    secrets:
+      DOCKERHUB_USER:
+        description: "DOCKERHUB_USER"
+        required: true
+      DOCKERHUB_TOKEN:
+        description: "DOCKERHUB_TOKEN"
+        required: true
+    inputs:
+      ERA_BELLMAN_CUDA_RELEASE:
+        description: "ERA_BELLMAN_CUDA_RELEASE"
+        type: string
+        required: true
+      image_tag_suffix:
+        description: "Optional suffix to override tag name generation"
+        type: string
+        required: false
+      action:
+        description: "Action with docker image"
+        type: string
+        default: "push"
+        required: false
+      is_pr_from_fork:
+        description: "Indicates whether the workflow is invoked from a PR created from fork"
+        type: boolean
+        default: false
+        required: false
+      CUDA_ARCH:
+        description: "CUDA Arch to build"
+        type: string
+        default: "89"
+        required: false
+      WITNESS_GENERATOR_RUST_FLAGS:
+        description: "Rust flags for witness_generator compilation"
+        type: string
+        default: ""
+        required: false
+    outputs:
+      protocol_version:
+        description: "Protocol version of the binary"
+        value: ${{ jobs.build-images.outputs.protocol_version }}
+
+jobs:
+  build-images:
+    name: Build and Push Docker Images
+    env:
+      IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
+      RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
+      ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
+      CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
+      WITNESS_GENERATOR_RUST_FLAGS: ${{ inputs.WITNESS_GENERATOR_RUST_FLAGS }}
+    runs-on: [ matterlabs-ci-runner-c3d ]
+    strategy:
+      matrix:
+        component:
+          - witness-generator
+    outputs:
+      protocol_version: ${{ steps.protocolversion.outputs.protocol_version }}
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+        with:
+          submodules: "recursive"
+
+      - name: setup-env
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo CI=1 >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo CI=1 >> .env
+          echo IN_DOCKER=1 >> .env
+
+      - name: start-services
+        run: |
+          echo "IMAGE_TAG_SUFFIX=${{ env.IMAGE_TAG_SUFFIX }}" >> .env
+          mkdir -p ./volumes/postgres
+          run_retried docker compose pull zk postgres
+          docker compose up -d zk postgres
+          ci_run sccache --start-server
+
+      - name: init
+        run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts
+          ci_run zk
+
+      - name: download CRS for GPU compressor
+        if: matrix.component == 'proof-fri-gpu-compressor'
+        run: |
+          ci_run run_retried curl -LO https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2\^24.key
+
+
+      - name: login to Docker registries
+        if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/'))
+        run: |
+          ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
+          ci_run gcloud auth configure-docker us-docker.pkg.dev -q
+
+      # We need to run this only when ERA_BELLMAN_CUDA_RELEASE is not available
+      # In our case it happens only when PR is created from fork
+      - name: Wait for runner IP to be not rate-limited against GH API
+        if: inputs.is_pr_from_fork == true
+        run: |
+          api_endpoint="https://api.github.com/users/zksync-era-bot"
+          wait_time=60
+          max_retries=60
+          retry_count=0
+
+          while [[ $retry_count -lt $max_retries ]]; do
+            response=$(run_retried curl -s -w "%{http_code}" -o temp.json "$api_endpoint")
+            http_code=$(echo "$response" | tail -n1)
+
+            if [[ "$http_code" == "200" ]]; then
+              echo "Request successful. Not rate-limited."
+              cat temp.json
+              rm temp.json
+              exit 0
+            elif [[ "$http_code" == "403" ]]; then
+              rate_limit_exceeded=$(jq -r '.message' temp.json | grep -i "API rate limit exceeded")
+              if [[ -n "$rate_limit_exceeded" ]]; then
+                retry_count=$((retry_count+1))
+                echo "API rate limit exceeded. Retry $retry_count of $max_retries. Retrying in $wait_time seconds..."
+                sleep $wait_time
+              else
+                echo "Request failed with HTTP status $http_code."
+                cat temp.json
+                rm temp.json
+                exit 1
+              fi
+            else
+              echo "Request failed with HTTP status $http_code."
+              cat temp.json
+              rm temp.json
+              exit 1
+            fi
+          done
+
+          echo "Reached the maximum number of retries ($max_retries). Exiting."
+          rm temp.json
+          exit 1
+
+      - name: protocol-version
+        id: protocolversion
+        # TODO: use -C flag, when it will become stable.
+        shell: bash
+        run: |
+          ci_run bash -c "cd prover && cargo build --release --bin prover_version"
+          PPV=$(ci_run prover/target/release/prover_version)
+          echo Protocol version is ${PPV}
+          echo "protocol_version=${PPV}" >> $GITHUB_OUTPUT
+          echo "PROTOCOL_VERSION=${PPV}" >> $GITHUB_ENV
+
+      - name: setup-rust-flags-env
+        if: matrix.component == 'witness-generator'
+        run: |
+          echo RUST_FLAGS="${{ env.WITNESS_GENERATOR_RUST_FLAGS }}" >> $GITHUB_ENV
+
+      - name: update-images
+        env:
+          DOCKER_ACTION: ${{ inputs.action }}
+          COMPONENT: ${{ matrix.component }}
+        run: |
+          PASSED_ENV_VARS="ERA_BELLMAN_CUDA_RELEASE,CUDA_ARCH,PROTOCOL_VERSION,RUST_FLAGS" \
+          ci_run zk docker $DOCKER_ACTION $COMPONENT
+
+      - name: Show sccache stats
+        if: always()
+        run: |
+          ci_run sccache --show-stats || true
+          ci_run cat /tmp/sccache_log.txt || true

.github/workflows/ci-core-reusable.yml

@@ -63,7 +63,11 @@ jobs:
         run: ci_run yarn l1-contracts test
 
       - name: Rust unit tests
-        run: ci_run zk test rust
+        run: |
+          ci_run zk test rust
+          # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
+          # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
+          ci_run zk f cargo test --release -p vm-benchmark --bench criterion --bench fill_bootloader
 
   loadtest:
     runs-on: [matterlabs-ci-runner]

.github/workflows/ci.yml

@@ -166,6 +166,21 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
+  build-witness-generator-image-avx512:
+    name: Build prover images with avx512 instructions
+    needs: changed_files
+    if: ${{ (needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }}
+    uses: ./.github/workflows/build-witness-generator-template.yml
+    with:
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
+      action: "build"
+      ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
+      is_pr_from_fork: ${{ github.event.pull_request.head.repo.fork == true }}
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
   ci-success:
     name: Github Status Check
     runs-on: ubuntu-latest

.github/workflows/release-test-stage.yml

@@ -106,6 +106,20 @@ jobs:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
+  build-push-witness-generator-image-avx512:
+    name: Build and push prover images with avx512 instructions
+    needs: [setup, changed_files]
+    uses: ./.github/workflows/build-witness-generator-template.yml
+    if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
+    with:
+      image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
+      ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
+      CUDA_ARCH: "60;70;75;89"
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
+    secrets:
+      DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+
   build-gar-prover-fri-gpu:
     name: Build GAR prover FRI GPU
     needs: [setup, build-push-prover-images]

.github/workflows/zk-environment-publish.yml

@@ -129,7 +129,7 @@ jobs:
       packages: write
       contents: read
     needs: [changed_files, get_short_sha, zk_environment]
-    runs-on: ubuntu-latest
+    runs-on: matterlabs-ci-runner
     steps:
       - name: Login to DockerHub
         uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0

.prettierignore

@@ -2,6 +2,35 @@
 bellman-cuda
 sdk/zksync-rs/CHANGELOG.md
 CHANGELOG.md
+core/lib/dal/.sqlx
+prover/lib/dal/.sqlx
+node_modules
 
 # Ignore contract submodules
 contracts
+
+**/target/**
+**/node_modules
+volumes
+**/build/**
+dist
+.git
+generated
+grafonnet-lib
+prettier-config
+lint-config
+**/cache
+**/artifacts
+**/typechain
+binaryen
+system-contracts
+artifacts-zk
+cache-zk
+# Ignore directories with OZ and forge submodules.
+contracts/l1-contracts/lib
+
+# Ignore ecosystem configs
+/configs/
+
+**/.git
+**/node_modules