Merge branch 'main' into sanekmelnikov/zk-inception-ecosystem-only

matter-labs · Oct 1, 2024 · 006f827 · 006f827
2 parents dd5d651 + 3b69e37
commit 006f827
Show file tree

Hide file tree

Showing 258 changed files with 8,921 additions and 5,477 deletions.
diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json
@@ -1,5 +1,5 @@
 {
-  "core": "24.26.0",
+  "core": "24.27.0",
   "prover": "16.5.0",
   "zk_toolbox": "0.1.2"
 }
diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml
@@ -71,15 +71,11 @@ jobs:
             if [ $(jq length <<<"$tags") -eq 0 ]; then
               echo "No tag found on all pages."
               echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV"
-              # TODO Remove it when we migrate to foundry inside contracts repository
-              mkdir -p contracts/l1-contracts/artifacts/ 
               exit 0
             fi
             filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags")
             if [[ ! -z "$filtered_tag" ]]; then
               echo "BUILD_CONTRACTS=false" >> "$GITHUB_ENV"
-              # TODO Remove it when we migrate to foundry inside contracts repository
-              mkdir -p contracts/l1-contracts/out
               break
             fi
             ((page++))

diff --git a/.github/workflows/build-docker-from-tag.yml b/.github/workflows/build-docker-from-tag.yml
@@ -103,15 +103,15 @@ jobs:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: "60;70;75;80;89"
-      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl --cfg=no_cuda"
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-gar-prover-fri-gpu:
+  build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
     name: Build GAR prover FRI GPU
     needs: [ setup, build-push-prover-images ]
-    uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
+    uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
     if: contains(github.ref_name, 'prover')
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}

diff --git a/...ub/workflows/build-prover-fri-gpu-gar.yml → ...ri-gpu-gar-and-circuit-prover-gpu-gar.yml b/...ub/workflows/build-prover-fri-gpu-gar.yml → ...ri-gpu-gar-and-circuit-prover-gpu-gar.yml
@@ -1,4 +1,4 @@
-name: Build Prover FRI GPU with builtin setup data
+name: Build Prover FRI GPU & Circuit Prover GPU with builtin setup data
 
 on:
   workflow_call:
@@ -17,7 +17,7 @@ on:
         type: string
 
 jobs:
-  build-gar-prover-fri-gpu:
+  build:
     name: Build prover FRI GPU GAR
     runs-on: [matterlabs-ci-runner-high-performance]
     steps:
@@ -28,6 +28,7 @@ jobs:
       - name: Download Setup data
         run: |
           gsutil -m rsync -r gs://matterlabs-setup-data-us/${{ inputs.setup_keys_id }} docker/prover-gpu-fri-gar
+          cp -v docker/prover-gpu-fri-gar/*.bin docker/circuit-prover-gpu-gar/
 
       - name: Login to us-central1 GAR
         run: |
@@ -39,7 +40,15 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@988b5a0280414f521da01fcc63a27aeeb4b104db # v3.6.1
 
-      - name: Build and push
+      - name: Login to Asia GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
+
+      - name: Login to Europe GAR
+        run: |
+          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
+
+      - name: Build and push prover-gpu-fri-gar
         uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
         with:
           context: docker/prover-gpu-fri-gar
@@ -49,22 +58,36 @@ jobs:
           tags: |
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Login to Asia GAR
+      - name: Build and push prover-gpu-fri-gar to Asia GAR
         run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://asia-docker.pkg.dev
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push to Asia GAR
+      - name: Build and push prover-gpu-fri-gar to Europe GAR
         run: |
           docker buildx imagetools create \
-            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
             us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Login to Europe GAR
+      - name: Build and push circuit-prover-gpu-gar
+        uses: docker/build-push-action@5cd11c3a4ced054e52742c5fd54dca954e0edd85 # v6.7.0
+        with:
+          context: docker/circuit-prover-gpu-gar
+          build-args: |
+            PROVER_IMAGE=${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+          push: true
+          tags: |
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+
+      - name: Build and push circuit-prover-gpu-gar to Asia GAR
         run: |
-          gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://europe-docker.pkg.dev
+          docker buildx imagetools create \
+            --tag asia-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
 
-      - name: Build and push to Europe GAR
+      - name: Build and push circuit-prover-gpu-gar to Europe GAR
         run: |
           docker buildx imagetools create \
-            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
-            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/prover-fri-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
+            --tag europe-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }} \
+            us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/circuit-prover-gpu-gar:2.0-${{ inputs.protocol_version }}-${{ inputs.image_tag_suffix }}
diff --git a/.github/workflows/build-prover-template.yml b/.github/workflows/build-prover-template.yml
@@ -52,6 +52,7 @@ jobs:
           - witness-generator
           - prover-gpu-fri
           - witness-vector-generator
+          - circuit-prover-gpu
           - prover-fri-gateway
           - prover-job-monitor
           - proof-fri-gpu-compressor

diff --git a/.github/workflows/build-witness-generator-template.yml b/.github/workflows/build-witness-generator-template.yml
@@ -51,6 +51,7 @@ jobs:
       ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: ${{ inputs.CUDA_ARCH }}
       WITNESS_GENERATOR_RUST_FLAGS: ${{ inputs.WITNESS_GENERATOR_RUST_FLAGS }}
+      ZKSYNC_USE_CUDA_STUBS: true
     runs-on: [ matterlabs-ci-runner-c3d ]
     strategy:
       matrix:

diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml
@@ -22,7 +22,7 @@ jobs:
           echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
           echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
           echo "RUSTC_WRAPPER=sccache" >> .env
-          echo "RUSTFLAGS=--cfg=no_cuda" >> .env
+          echo "ZKSYNC_USE_CUDA_STUBS=true" >> .env
 
       - name: Start services
         run: |

diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml
@@ -19,7 +19,7 @@ jobs:
           echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
           echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
           echo "RUSTC_WRAPPER=sccache" >> .env
-          echo "RUSTFLAGS=--cfg=no_cuda" >> .env
+          echo "ZKSYNC_USE_CUDA_STUBS=true" >> .env
           echo "prover_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local_prover" >> $GITHUB_ENV
           echo "core_url=postgres://postgres:notsecurepassword@localhost:5432/zksync_local" >> $GITHUB_ENV
 

diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml
@@ -185,6 +185,7 @@ jobs:
           SNAPSHOT_RECOVERY_LOGS_DIR=logs/snapshot_recovery/
           GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/
           EXTERNAL_NODE_LOGS_DIR=logs/external_node
+          FEES_LOGS_DIR=logs/fees
           REVERT_LOGS_DIR=logs/revert
 
           mkdir -p $SERVER_LOGS_DIR
@@ -193,6 +194,7 @@ jobs:
           mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR
           mkdir -p $GENESIS_RECOVERY_LOGS_DIR
           mkdir -p $EXTERNAL_NODE_LOGS_DIR
+          mkdir -p $FEES_LOGS_DIR
           mkdir -p $REVERT_LOGS_DIR
 
           echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV
@@ -201,6 +203,7 @@ jobs:
           echo "SNAPSHOT_RECOVERY_LOGS_DIR=$SNAPSHOT_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
           echo "GENESIS_RECOVERY_LOGS_DIR=$GENESIS_RECOVERY_LOGS_DIR" >> $GITHUB_ENV
           echo "EXTERNAL_NODE_LOGS_DIR=$EXTERNAL_NODE_LOGS_DIR" >> $GITHUB_ENV
+          echo "FEES_LOGS_DIR=$FEES_LOGS_DIR" >> $GITHUB_ENV
           echo "REVERT_LOGS_DIR=$REVERT_LOGS_DIR" >> $GITHUB_ENV
 
       - name: Initialize ecosystem
@@ -220,9 +223,9 @@ jobs:
 
       - name: Read Custom Token address and set as environment variable
         run: |
-          address=$(awk -F": " '/tokens:/ {found_tokens=1} found_tokens && /DAI:/ {found_dai=1} found_dai && /address:/ {print $2; exit}' ./configs/erc20.yaml)
-          echo "address=$address"
-          echo "address=$address" >> $GITHUB_ENV
+          CUSTOM_TOKEN_ADDRESS=$(awk -F": " '/tokens:/ {found_tokens=1} found_tokens && /DAI:/ {found_dai=1} found_dai && /address:/ {print $2; exit}' ./configs/erc20.yaml)
+          echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS"
+          echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS" >> $GITHUB_ENV
 
       - name: Create and initialize Validium chain
         run: |
@@ -245,7 +248,6 @@ jobs:
           --server-db-name=zksync_server_localhost_validium \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_validium \
-          --port-offset 2000 \
           --chain validium
 
       - name: Create and initialize chain with Custom Token
@@ -256,7 +258,7 @@ jobs:
           --prover-mode no-proofs \
           --wallet-creation localhost \
           --l1-batch-commit-data-generator-mode rollup \
-          --base-token-address ${{ env.address }} \
+          --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
           --base-token-price-nominator 3 \
           --base-token-price-denominator 2 \
           --set-as-default false \
@@ -269,7 +271,6 @@ jobs:
           --server-db-name=zksync_server_localhost_custom_token \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_custom_token \
-          --port-offset 3000 \
           --chain custom_token
 
       - name: Create and register chain with transactions signed "offline"
@@ -315,7 +316,7 @@ jobs:
           --prover-mode no-proofs \
           --wallet-creation localhost \
           --l1-batch-commit-data-generator-mode validium \
-          --base-token-address ${{ env.address }} \
+          --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \
           --base-token-price-nominator 3 \
           --base-token-price-denominator 2 \
           --set-as-default false \
@@ -328,7 +329,6 @@ jobs:
           --server-db-name=zksync_server_localhost_consensus \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_consensus \
-          --port-offset 4000 \
           --chain consensus
 
       - name: Build test dependencies
@@ -351,6 +351,10 @@ jobs:
 
           ci_run sleep 5
 
+      - name: Setup attester committee for the consensus chain
+        run: |
+          ci_run zk_inception consensus set-attester-committee --chain consensus &> ${{ env.INTEGRATION_TESTS_LOGS_DIR }}/consensus.log 
+
       - name: Run integration tests
         run: |
           PASSED_ENV_VARS="RUN_CONTRACT_VERIFICATION_TEST" \
@@ -454,6 +458,27 @@ jobs:
           wait $PID3
           wait $PID4
 
+      - name: Fee projection tests
+        run: |
+          ci_run killall -INT zksync_server || true
+          
+          ci_run zk_supervisor test fees --no-deps --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log &
+          PID1=$!
+          
+          ci_run zk_supervisor test fees --no-deps --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log &
+          PID2=$!
+          
+          ci_run zk_supervisor test fees --no-deps --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log &
+          PID3=$!
+          
+          ci_run zk_supervisor test fees --no-deps --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log &
+          PID4=$!
+          
+          wait $PID1
+          wait $PID2
+          wait $PID3
+          wait $PID4
+
       - name: Run revert tests
         run: |
           ci_run killall -INT zksync_server || true

diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml
@@ -57,7 +57,7 @@ jobs:
           echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env
           echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env
           echo "RUSTC_WRAPPER=sccache" >> .env
-          echo "RUSTFLAGS=--cfg=no_cuda" >> .env
+          echo "ZKSYNC_USE_CUDA_STUBS=true" >> .env
 
       - name: Start services
         run: |

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -166,7 +166,7 @@ jobs:
     with:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
       action: "build"
-      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl --cfg=no_cuda"
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl"
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}

diff --git a/.github/workflows/release-test-stage.yml b/.github/workflows/release-test-stage.yml
@@ -115,15 +115,15 @@ jobs:
       image_tag_suffix: ${{ needs.setup.outputs.image_tag_suffix }}-avx512
       ERA_BELLMAN_CUDA_RELEASE: ${{ vars.ERA_BELLMAN_CUDA_RELEASE }}
       CUDA_ARCH: "60;70;75;80;89"
-      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl --cfg=no_cuda"
+      WITNESS_GENERATOR_RUST_FLAGS: "-Ctarget_feature=+avx512bw,+avx512cd,+avx512dq,+avx512f,+avx512vl "
     secrets:
       DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
       DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
 
-  build-gar-prover-fri-gpu:
+  build-gar-prover-fri-gpu-and-circuit-prover-gpu-gar:
     name: Build GAR prover FRI GPU
     needs: [ setup, build-push-prover-images ]
-    uses: ./.github/workflows/build-prover-fri-gpu-gar.yml
+    uses: ./.github/workflows/build-prover-fri-gpu-gar-and-circuit-prover-gpu-gar.yml
     if: needs.changed_files.outputs.prover == 'true' || needs.changed_files.outputs.all == 'true'
     with:
       setup_keys_id: ${{ needs.setup.outputs.prover_fri_gpu_key_id }}