Merge branch 'main' into tee_prover_new_2

.github/workflows/ci.yml

@@ -42,6 +42,9 @@ jobs:
               - '!prover/extract-setup-data-keys.sh'
               - 'docker/prover*/**'
               - '.github/workflows/build-prover-template.yml'
+              - '.github/workflows/new-build-prover-template.yml'
+              - '.github/workflows/build-witness-generator-template.yml'
+              - '.github/workflows/new-build-witness-generator-template.yml'
               - '.github/workflows/ci-prover-reusable.yml'
               - 'docker-compose-runner-nightly.yml'
               - '!**/*.md'
@@ -53,7 +56,9 @@ jobs:
               - 'docker/external-node/**'
               - 'docker/server/**'
               - '.github/workflows/build-core-template.yml'
+              - '.github/workflows/new-build-core-template.yml'
               - '.github/workflows/build-contract-verifier-template.yml'
+              - '.github/workflows/new-build-contract-verifier-template.yml'
               - '.github/workflows/ci-core-reusable.yml'
               - '.github/workflows/ci-core-lint-reusable.yml'
               - 'Cargo.toml'

.github/workflows/new-build-contract-verifier-template.yml

@@ -74,36 +74,30 @@ jobs:
           tar -C ./contracts -zxf system-contracts.tar.gz
 
       - name: Install Apt dependencies
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config
 
       - name: Install Node
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
           cache: 'npm'
 
       - name: Install Yarn
+        if: env.BUILD_CONTRACTS == 'true'
         run: npm install -g yarn
 
       - name: Setup rust
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
         with:
           toolchain: nightly-2024-08-01
 
-      - name: Install cargo-nextest from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: cargo-nextest
-
-      - name: Install sqlx-cli from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: sqlx-cli
-          tag: 0.8.1
-
       - name: Install foundry-zksync
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           mkdir ./foundry-zksync
           curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
@@ -112,6 +106,7 @@ jobs:
           echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           # Download needed versions of vyper compiler
@@ -129,18 +124,14 @@ jobs:
             chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
           done
 
-      - name: init
-        shell: bash
-        run: |
-          mkdir -p ./volumes/postgres
-          docker compose up -d postgres
-
       - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
 
       - name: build contracts
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           cp etc/tokens/{test,localhost}.json

.github/workflows/new-build-core-template.yml

@@ -79,36 +79,30 @@ jobs:
           tar -C ./contracts -zxf system-contracts.tar.gz
 
       - name: Install Apt dependencies
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           sudo apt-get update && sudo apt-get install -y libssl-dev pkg-config
 
       - name: Install Node
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
         with:
           node-version: 20
           cache: 'npm'
 
       - name: Install Yarn
+        if: env.BUILD_CONTRACTS == 'true'
         run: npm install -g yarn
 
       - name: Setup rust
+        if: env.BUILD_CONTRACTS == 'true'
         uses: actions-rust-lang/setup-rust-toolchain@1fbea72663f6d4c03efaab13560c8a24cfd2a7cc # v1.9.0
         with:
           toolchain: nightly-2024-08-01
 
-      - name: Install cargo-nextest from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: cargo-nextest
-
-      - name: Install sqlx-cli from crates.io
-        uses: baptiste0928/cargo-install@904927dbe77864e0f2281519fe9d5bd097a220b3 # v3.1.1
-        with:
-          crate: sqlx-cli
-          tag: 0.8.1
-
       - name: Install foundry-zksync
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           mkdir ./foundry-zksync
           curl -LO https://github.com/matter-labs/foundry-zksync/releases/download/nightly-15bec2f861b3b4c71e58f85e2b2c9dd722585aa8/foundry_nightly_linux_amd64.tar.gz
@@ -117,6 +111,7 @@ jobs:
           echo "$PWD/foundry-zksync" >> $GITHUB_PATH
 
       - name: Pre-download compilers
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           # Download needed versions of vyper compiler
@@ -134,18 +129,14 @@ jobs:
             chmod +x "./hardhat-nodejs/compilers-v2/$compiler/${compiler}-v${version}"
           done
 
-      - name: init
-        shell: bash
-        run: |
-          mkdir -p ./volumes/postgres
-          docker compose up -d postgres
-
       - name: Install zkstack
+        if: env.BUILD_CONTRACTS == 'true'
         run: |
           ./zkstack_cli/zkstackup/install --path ./zkstack_cli/zkstackup/zkstackup
           zkstackup --local || true
 
       - name: build contracts
+        if: env.BUILD_CONTRACTS == 'true'
         shell: bash
         run: |
           cp etc/tokens/{test,localhost}.json

core/lib/config/src/configs/prover_autoscaler.rs

@@ -51,6 +51,8 @@ pub struct ProverAutoscalerScalerConfig {
     pub cluster_priorities: HashMap<String, u32>,
     /// Prover speed per GPU. Used to calculate desired number of provers for queue size.
     pub prover_speed: HashMap<Gpu, u32>,
+    /// Maximum number of provers which can be run per cluster/GPU.
+    pub max_provers: HashMap<String, HashMap<Gpu, u32>>,
     /// Duration after which pending pod considered long pending.
     #[serde(default = "ProverAutoscalerScalerConfig::default_long_pending_duration")]
     pub long_pending_duration: Duration,

core/lib/dal/src/blocks_web3_dal.rs

@@ -5,6 +5,7 @@ use zksync_db_connection::{
 use zksync_system_constants::EMPTY_UNCLES_HASH;
 use zksync_types::{
     api,
+    debug_flat_call::CallTraceMeta,
     fee_model::BatchFeeInput,
     l2_to_l1_log::L2ToL1Log,
     web3::{BlockHeader, Bytes},
@@ -531,26 +532,33 @@ impl BlocksWeb3Dal<'_, '_> {
     pub async fn get_traces_for_l2_block(
         &mut self,
         block_number: L2BlockNumber,
-    ) -> DalResult<Vec<(Call, H256, usize)>> {
-        let protocol_version = sqlx::query!(
+    ) -> DalResult<Vec<(Call, CallTraceMeta)>> {
+        let row = sqlx::query!(
             r#"
             SELECT
-                protocol_version
+                protocol_version,
+                hash
             FROM
                 miniblocks
             WHERE
                 number = $1
             "#,
             i64::from(block_number.0)
         )
-        .try_map(|row| row.protocol_version.map(parse_protocol_version).transpose())
+        .try_map(|row| {
+            row.protocol_version
+                .map(parse_protocol_version)
+                .transpose()
+                .map(|val| (val, H256::from_slice(&row.hash)))
+        })
         .instrument("get_traces_for_l2_block#get_l2_block_protocol_version_id")
         .with_arg("l2_block_number", &block_number)
         .fetch_optional(self.storage)
         .await?;
-        let Some(protocol_version) = protocol_version else {
+        let Some((protocol_version, block_hash)) = row else {
             return Ok(Vec::new());
         };
+
         let protocol_version =
             protocol_version.unwrap_or_else(ProtocolVersionId::last_potentially_undefined);
 
@@ -577,9 +585,15 @@ impl BlocksWeb3Dal<'_, '_> {
         .await?
         .into_iter()
         .map(|call_trace| {
-            let hash = H256::from_slice(&call_trace.tx_hash);
+            let tx_hash = H256::from_slice(&call_trace.tx_hash);
             let index = call_trace.tx_index_in_block.unwrap_or_default() as usize;
-            (call_trace.into_call(protocol_version), hash, index)
+            let meta = CallTraceMeta {
+                index_in_block: index,
+                tx_hash,
+                block_number: block_number.0,
+                block_hash,
+            };
+            (call_trace.into_call(protocol_version), meta)
         })
         .collect())
     }
@@ -1105,9 +1119,9 @@ mod tests {
             .await
             .unwrap();
         assert_eq!(traces.len(), 2);
-        for ((trace, hash, _index), tx_result) in traces.iter().zip(&tx_results) {
+        for ((trace, meta), tx_result) in traces.iter().zip(&tx_results) {
             let expected_trace = tx_result.call_trace().unwrap();
-            assert_eq!(&tx_result.hash, hash);
+            assert_eq!(tx_result.hash, meta.tx_hash);
             assert_eq!(*trace, expected_trace);
         }
     }

core/lib/dal/src/transactions_dal.rs

@@ -10,9 +10,10 @@ use zksync_db_connection::{
     utils::pg_interval_from_duration,
 };
 use zksync_types::{
-    block::L2BlockExecutionData, l1::L1Tx, l2::L2Tx, protocol_upgrade::ProtocolUpgradeTx, Address,
-    ExecuteTransactionCommon, L1BatchNumber, L1BlockNumber, L2BlockNumber, PriorityOpId,
-    ProtocolVersionId, Transaction, H256, PROTOCOL_UPGRADE_TX_TYPE, U256,
+    block::L2BlockExecutionData, debug_flat_call::CallTraceMeta, l1::L1Tx, l2::L2Tx,
+    protocol_upgrade::ProtocolUpgradeTx, Address, ExecuteTransactionCommon, L1BatchNumber,
+    L1BlockNumber, L2BlockNumber, PriorityOpId, ProtocolVersionId, Transaction, H256,
+    PROTOCOL_UPGRADE_TX_TYPE, U256,
 };
 use zksync_utils::u256_to_big_decimal;
 use zksync_vm_interface::{
@@ -2131,12 +2132,17 @@ impl TransactionsDal<'_, '_> {
         Ok(data)
     }
 
-    pub async fn get_call_trace(&mut self, tx_hash: H256) -> DalResult<Option<(Call, usize)>> {
+    pub async fn get_call_trace(
+        &mut self,
+        tx_hash: H256,
+    ) -> DalResult<Option<(Call, CallTraceMeta)>> {
         let row = sqlx::query!(
             r#"
             SELECT
                 protocol_version,
-                index_in_block
+                index_in_block,
+                miniblocks.number AS "miniblock_number!",
+                miniblocks.hash AS "miniblocks_hash!"
             FROM
                 transactions
             INNER JOIN miniblocks ON transactions.miniblock_number = miniblocks.number
@@ -2177,7 +2183,12 @@ impl TransactionsDal<'_, '_> {
         .map(|call_trace| {
             (
                 parse_call_trace(&call_trace.call_trace, protocol_version),
-                row.index_in_block.unwrap_or_default() as usize,
+                CallTraceMeta {
+                    index_in_block: row.index_in_block.unwrap_or_default() as usize,
+                    tx_hash,
+                    block_number: row.miniblock_number as u32,
+                    block_hash: H256::from_slice(&row.miniblocks_hash),
+                },
             )
         }))
     }