Merge branch 'main' into aov-pla-1026-fix-missing-revert-data-error

matter-labs · Sep 10, 2024 · b6aaf97 · b6aaf97
2 parents 1f28c38 + 92dde03
commit b6aaf97
Show file tree

Hide file tree

Showing 235 changed files with 4,969 additions and 1,992 deletions.
diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,5 +1,5 @@
 {
-  "core": "24.23.0",
+  "core": "24.24.0",
   "prover": "16.5.0",
   "zk_toolbox": "0.1.2"
 }
diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml
@@ -23,7 +23,7 @@ concurrency: docker-build
 jobs:
   setup:
     name: Setup
-    runs-on: [ubuntu-latest]
+    runs-on: [ ubuntu-latest ]
     outputs:
       image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }}
       prover_fri_gpu_key_id: ${{ steps.extract-prover-fri-setup-key-ids.outputs.gpu_short_commit_sha }}
@@ -48,7 +48,7 @@ jobs:
 
   build-push-core-images:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-core-template.yml
     if: contains(github.ref_name, 'core')
     secrets:
@@ -60,7 +60,7 @@ jobs:
 
   build-push-tee-prover-images:
     name: Build and push images
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-tee-prover-template.yml
     if: contains(github.ref_name, 'core')
     secrets:
@@ -72,7 +72,7 @@ jobs:
 
   build-push-contract-verifier:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-contract-verifier-template.yml
     if: contains(github.ref_name, 'contract_verifier')
     secrets:
@@ -83,20 +83,20 @@ jobs:
 
   build-push-prover-images:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-prover-template.yml
     if: contains(github.ref_name, 'prover')
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
-      CUDA_ARCH: "60;70;75;89"
+      CUDA_ARCH: "60;70;75;80;89"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
   build-push-witness-generator-image-avx512:
     name: Build and push image
-    needs: [setup]
+    needs: [ setup ]
     uses: ./.github/workflows/build-witness-generator-template.yml
     if: contains(github.ref_name, 'prover')
     with:
@@ -110,7 +110,7 @@ jobs:
 
   build-gar-prover-fri-gpu:
     name: Build GAR prover FRI GPU
-    needs: [setup, build-push-prover-images]
+    needs: [ setup, build-push-prover-images ]
     uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
     if: contains(github.ref_name, 'prover')
     with:

diff --git a/.github/workflows/build-prover-fri-gpu-gar.yml b/.github/workflows/build-prover-fri-gpu-gar.yml
@@ -34,13 +34,13 @@ jobs:
           gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:
           context: docker/prover-gpu-fri-gar
           build-args: |

diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -180,7 +180,7 @@ jobs:
           - witness-vector-generator
     steps:
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
       - name: Login to us-central1 GAR
         run: |

diff --git a/.github/workflows/build-tee-prover-template.yml b/.github/workflows/build-tee-prover-template.yml
@@ -28,15 +28,15 @@ jobs:
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
     runs-on: [matterlabs-ci-runner-high-performance]
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         if: ${{ github.event_name == 'workflow_dispatch' }}
         with:
           ref: ${{ github.event.inputs.target_branch }}
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         if: ${{ github.event_name != 'workflow_dispatch' }}
 
-      - uses: cachix/install-nix-action@v27
+      - uses: cachix/install-nix-action@ba0dd844c9180cbf77aa72a116d6fbc515d0e87b # v27
         with:
           extra_nix_config: |
             access-tokens = github.com=${{ github.token }}
@@ -45,7 +45,7 @@ jobs:
             sandbox = true
 
       - name: Setup Attic cache
-        uses: ryanccn/attic-action@v0
+        uses: ryanccn/attic-action@618a980988d704a7709daeea88526146acd1d45f # v0.2.1
         with:
           endpoint: https://attic.teepot.org/
           cache: tee-pot

diff --git a/.github/workflows/cargo-license.yaml b/.github/workflows/cargo-license.yaml
@@ -6,3 +6,6 @@ jobs:
     steps:
     - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
     - uses: EmbarkStudios/cargo-deny-action@8371184bd11e21dcf8ac82ebf8c9c9f74ebf7268 # v2.0.1
+      with:
+        command: check
+        command-arguments: "--hide-inclusion-graph"
diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -52,22 +52,19 @@ jobs:
 
       - name: Init
         run: |
-          ci_run zk
           ci_run run_retried rustup show
-          ci_run zk run yarn
-          ci_run zk db setup
-          ci_run zk compiler all
-          ci_run zk contract build
+          ci_run ./bin/zkt
+          ci_run zk_supervisor contracts
 
       - name: Contracts unit tests
         run: ci_run yarn l1-contracts test
 
       - name: Rust unit tests
         run: |
-          ci_run zk test rust
+          ci_run zk_supervisor test rust
           # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
           # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
-          ci_run zk f cargo test --release -p vm-benchmark --bench oneshot --bench batch
+          ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch
 
   loadtest:
     runs-on: [ matterlabs-ci-runner-high-performance ]
@@ -90,8 +87,10 @@ jobs:
 
       - name: Loadtest configuration
         run: |
-          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 22000 || 16000 }} >> .env
-          echo ACCOUNTS_AMOUNT="150" >> .env
+          echo EXPECTED_TX_COUNT=${{ matrix.vm_mode == 'new' && 21000 || 16000 }} >> .env
+          echo ACCOUNTS_AMOUNT="100" >> .env
+          echo MAX_INFLIGHT_TXS="10" >> .env
+          echo SYNC_API_REQUESTS_LIMIT="15" >> .env
           echo FAIL_FAST=true >> .env
           echo IN_DOCKER=1 >> .env
           echo DATABASE_MERKLE_TREE_MODE=lightweight >> .env
@@ -115,7 +114,8 @@ jobs:
       - name: Run server
         run: |
           EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE=${{ matrix.vm_mode }} \
-          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE" \
+          CHAIN_MEMPOOL_DELAY_INTERVAL=50 \
+          PASSED_ENV_VARS="EXPERIMENTAL_VM_STATE_KEEPER_FAST_VM_MODE,CHAIN_MEMPOOL_DELAY_INTERVAL" \
             ci_run zk server --uring --components api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads &>server.log &
           ci_run sleep 60
 

diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml
@@ -4,14 +4,19 @@ on:
 
 env:
   CLICOLOR: 1
+  # We run multiple binaries in parallel, and by default they will try to utilize all the
+  # available CPUs. In tests, there is not much CPU-intensive work (rayon), but a lot of
+  # async work (tokio), so we prioritize tokio.
+  TOKIO_WORKER_THREADS: 4
+  RAYON_NUM_THREADS: 2
 
 jobs:
   lint:
     name: lint
     uses: ./.github/workflows/ci-core-lint-reusable.yml
 
   tests:
-    runs-on: [ matterlabs-ci-runner ]
+    runs-on: [ matterlabs-ci-runner-ultra-performance ]
     steps:
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
         with:
@@ -249,7 +254,7 @@ jobs:
 
 
       - name: Upload logs
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
         if: always()
         with:
           name: logs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -31,7 +31,7 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             prover:

diff --git a/.github/workflows/nodejs-license.yaml b/.github/workflows/nodejs-license.yaml
@@ -47,7 +47,7 @@ jobs:
         uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
 
       - name: Use Node.js
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 18
 

diff --git a/.github/workflows/protobuf.yaml b/.github/workflows/protobuf.yaml
@@ -33,7 +33,7 @@ jobs:
   compatibility:
     runs-on: [ubuntu-22.04-github-hosted-16core]
     steps:
-      - uses: mozilla-actions/[email protected].3
+      - uses: mozilla-actions/sccache-action@89e9040de88b577a072e3760aaf59f585da083af # v0.0.5
 
       # before
       - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
@@ -69,7 +69,7 @@ jobs:
           | xargs cat > ./after.binpb
 
       # compare
-      - uses: bufbuild/buf-setup-action@v1
+      - uses: bufbuild/buf-setup-action@54abbed4fe8d8d45173eca4798b0c39a53a7b658 # v1.39.0
         with:
           github_token: ${{ github.token }}
       - name: buf breaking

diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml
@@ -15,7 +15,7 @@ jobs:
     steps:
       - name: Run release-please
         id: release
-        uses: google-github-actions/release-please-action@v4
+        uses: google-github-actions/release-please-action@e4dc86ba9405554aeba3c6bb2d169500e7d3b4ee # v4.1.1
         with:
             token: ${{ secrets.RELEASE_TOKEN }}
             config-file: .github/release-please/config.json

diff --git a/.github/workflows/release-test-stage.yml b/.github/workflows/release-test-stage.yml
@@ -23,7 +23,7 @@ jobs:
 
       - name: Get all test, doc and src files that have changed
         id: changed-files-yaml
-        uses: tj-actions/changed-files@v37
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             # TODO: make it more granular, as already implemented in CI workflow

diff --git a/.github/workflows/vm-perf-comparison.yml b/.github/workflows/vm-perf-comparison.yml
@@ -68,7 +68,7 @@ jobs:
         id: comparison
 
       - name: Comment on PR
-        uses: thollander/actions-comment-pull-request@v2
+        uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6 # v2.5.0
         with:
           message: |
             ${{ steps.comparison.outputs.speedup == '' && '## No performance difference detected (anymore)' || '## Detected VM performance changes' }}

diff --git a/.github/workflows/zk-environment-publish.yml b/.github/workflows/zk-environment-publish.yml
@@ -36,7 +36,7 @@ jobs:
 
       - name: Get changed files
         id: changed-files-yaml
-        uses: tj-actions/changed-files@v39
+        uses: tj-actions/changed-files@fea790cb660e33aef4bdf07304e28fedd77dfa13 # v39
         with:
           files_yaml: |
             zk_env: