Merge branch 'main' into mirror-deniallugo-vm-perf-optimization-008df…

…6deb6bd627ec9dc901a78aca374f16fd14e

.github/release-please/manifest.json

@@ -1,6 +1,6 @@
 {
   "sdk/zksync-web3.js": "0.15.4",
   "sdk/zksync-rs": "0.4.0",
-  "core": "15.0.2",
+  "core": "15.1.1",
   "prover": "7.1.1"
 }

.github/workflows/build-docker-from-tag.yml

@@ -1,5 +1,17 @@
 name: Build Image from tag
 on:
+  workflow_call:
+    inputs:
+      tag_name:
+        description: "Tag of an image to built"
+        type: string
+        required: true
+  workflow_dispatch:
+    inputs:
+      tag_name:
+        description: "Tag of an image to built"
+        type: string
+        required: true
   push:
     tags:
       - core-v**
@@ -10,7 +22,7 @@ concurrency: docker-build
 jobs:
   setup:
     name: Setup
-    runs-on: [k8s, stage]
+    runs-on: [ubuntu-latest]
     outputs:
       image_tag_suffix: ${{ steps.set.outputs.image_tag_suffix }}
     steps:
@@ -21,7 +33,12 @@ jobs:
       - name: Generate output with git tag
         id: set
         run: |
-          git_tag="${GITHUB_REF#refs/*/}"
+          git_tag=""
+          if [[ -z "${{ inputs.tag_name }}" ]]; then
+            git_tag="${GITHUB_REF#refs/*/}"
+          else
+            git_tag="${{ inputs.tag_name }}"
+          fi
           version=$(cut -d "-" -f2 <<< ${git_tag})
           echo "image_tag_suffix=${version}" >> $GITHUB_OUTPUT
 

.github/workflows/build-external-node-docker.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   build-images:
     name: External Node - Build and Push Docker Image
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:

.github/workflows/build-gar-reusable.yml

@@ -20,7 +20,7 @@ on:
 jobs:
   build-gar-prover:
     name: Build GAR prover
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
     strategy:
       fail-fast: false
       matrix:

.github/workflows/build-local-node-docker.yml

@@ -11,7 +11,7 @@ on:
 jobs:
   build-images:
     name: Local Node - Build and Push Docker Image
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:

.github/workflows/build-prover-fri-gpu-gar.yml

@@ -15,7 +15,7 @@ on:
 jobs:
   build-gar-prover-fri-gpu:
     name: Build prover FRI GPU GAR
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:

.github/workflows/build-prover-template.yml

@@ -28,26 +28,14 @@ on:
         required: false
 
 jobs:
-  era-bellman-cuda:
-    name: Get era-bellman-cuda release URLs.
-    runs-on: [ubuntu-latest]
-    outputs:
-      source: ${{ steps.release.outputs.source }}
-      binary: ${{ steps.release.outputs.binary }}
-    steps:
-      - id: release
-        run: |
-          release=($(curl -v --silent https://api.github.com/repos/matter-labs/era-bellman-cuda/releases/tags/${{ inputs.ERA_BELLMAN_CUDA_RELEASE }} | jq -r ".name, .tarball_url, .assets[0].browser_download_url"))
-          echo "source=${release[1]}" >> "$GITHUB_OUTPUT"
-          echo "binary=${release[2]}" >> "$GITHUB_OUTPUT"
-
   build-images:
     name: Build and Push Docker Images
     env:
       image_tag: ${{ inputs.image_tag }}
       IMAGE_TAG_SUFFIX: ${{ inputs.image_tag_suffix }}
+      RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
+      ERA_BELLMAN_CUDA_RELEASE: ${{ inputs.ERA_BELLMAN_CUDA_RELEASE }}
     runs-on: [matterlabs-ci-runner]
-    needs: [era-bellman-cuda]
     strategy:
       matrix:
         component:
@@ -64,9 +52,6 @@ jobs:
         with:
           submodules: "recursive"
 
-      - name: Prepare bellman-cuda directory
-        run: etc/scripts/prepare_era_bellman_cuda.sh ${{ needs.era-bellman-cuda.outputs.source }} ${{ needs.era-bellman-cuda.outputs.binary }}
-
       - name: setup-env
         run: |
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
@@ -112,7 +97,7 @@ jobs:
               "proof-fri-compressor"
             )
         run: |
-          nightly_components=${{env.RUST_NIGHTLY_COMPONENTS}}
+          nightly_components=${{ env.RUST_NIGHTLY_COMPONENTS }}
           ci_run docker login -u ${{ secrets.DOCKERHUB_USER }} -p ${{ secrets.DOCKERHUB_TOKEN }}
           ci_run gcloud auth configure-docker us-docker.pkg.dev,asia-docker.pkg.dev -q
 
@@ -126,10 +111,10 @@ jobs:
             if [ "$underscored_name" == "prover_gpu_fri" ]; then
               underscored_name="prover_fri"
             fi
-            ci_run rustup default nightly-2023-07-21
             ci_run echo [workspace] > Cargo.toml
             ci_run echo members = [\"prover/${underscored_name}\"] >> Cargo.toml
             ci_run cp prover/Cargo.lock Cargo.lock
+            PASSED_ENV_VARS="ERA_BELLMAN_CUDA_RELEASE"  \
             ci_run zk docker $DOCKER_ACTION $COMPONENT
           else
             ci_run zk docker $DOCKER_ACTION $COMPONENT

.github/workflows/ci-prover-reusable.yml

@@ -5,51 +5,33 @@ on:
 jobs:
   unit-tests:
     runs-on: [matterlabs-ci-runner]
+    env:
+      RUNNER_COMPOSE_FILE: "docker-compose-runner-nightly.yml"
 
     steps:
-      - name: Prepare environment
-        run: |
-          sudo apt update && sudo apt install -y \
-            pkg-config libclang-dev build-essential lldb lld \
-            clang openssl libssl-dev gcc g++ pkg-config libclang-dev curl wget
-
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
         with:
           submodules: "recursive"
 
-      - name: Use Node.js 18.18.0
-        uses: actions/setup-node@v2
-        with:
-          node-version: '18.18.0'
-
-      - name: Install Rust nightly-2023-07-21
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly-2023-07-21
-          override: true
-
       - name: Setup environment
         run: |
           echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
-          npm install -g yarn
-          cargo install sqlx-cli --version 0.5.13
 
       - name: Start services
         run: |
-          docker-compose -f docker-compose-runner.yml pull
-          docker-compose -f docker-compose-runner.yml up --build -d zk
+          docker-compose -f ${RUNNER_COMPOSE_FILE} pull
+          docker-compose -f ${RUNNER_COMPOSE_FILE} up --build -d geth zk postgres
           ci_run sccache --start-server
 
       - name: Init
         run: |
-          zk
-          zk config compile
-          zk db setup
+          ci_run zk
+          ci_run zk config compile
+          ci_run zk db setup
 
       - name: Prover unit tests
         run: |
-          cd prover
           # Not all tests are enabled since, prover and setup_key_generator_and_server requires bellman-cuda to be present
-          zk f cargo +nightly-2023-07-21 test --release -p zksync_witness_generator -p vk_setup_data_generator_server_fri -p zksync_prover_fri -p zksync_witness_vector_generator -p zksync_prover_fri_utils
+          ci_run bash -c "cd prover && zk f cargo test --release -p zksync_witness_generator -p vk_setup_data_generator_server_fri -p zksync_prover_fri -p zksync_witness_vector_generator -p zksync_prover_fri_utils"

.github/workflows/ci.yml

@@ -41,6 +41,9 @@ jobs:
             all:
               - '!core/**'
               - '!prover/**'
+              - '!.github/workflows/zk-environment.publish.yml'
+              - '!docker/zk-environment/Dockerfile'
+              - '!docker/zk-environment-cuda-12-0/Dockerfile'
   ci-for-core:
     name: CI for Core Components
     needs: changed_files

.github/workflows/coverage.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   generate:
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
 
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3

.github/workflows/release-please.yml

@@ -3,6 +3,10 @@ on:
     branches:
       - main
 
+permissions:
+  contents: write
+  pull-requests: write
+
 name: release-please
 jobs:
   release-please:
@@ -14,6 +18,7 @@ jobs:
         id: release
         uses: google-github-actions/release-please-action@v3
         with:
+            token: ${{ secrets.RELEASE_TOKEN }}
             command: manifest
             config-file: .github/release-please/config.json
             manifest-file: .github/release-please/manifest.json

.github/workflows/secrets_scanner.yaml

@@ -1,5 +1,7 @@
 name: Leaked Secrets Scan
-on: [pull_request]
+on:
+  pull_request:
+  merge_group:
 jobs:
   TruffleHog:
     runs-on: ubuntu-latest

.github/workflows/vm-perf-comparison.yml

@@ -6,7 +6,7 @@ on:
 jobs:
   vm-benchmarks:
     name: Run VM benchmarks
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/vm-perf-to-prometheus.yml

@@ -12,7 +12,7 @@ concurrency: vm-benchmarks
 jobs:
   vm-benchmarks:
     name: Run VM benchmarks
-    runs-on: [self-hosted, ci-runner]
+    runs-on: [matterlabs-ci-runner]
 
     steps:
       - uses: actions/checkout@v3