Merge branch 'main' into deniallugo-allow-zk-toolbox-find-ecosystem-i…

…n-parent-dirs

.github/release-please/manifest.json

@@ -1,4 +1,4 @@
 {
   "core": "24.9.0",
-  "prover": "15.1.0"
+  "prover": "16.0.0"
 }

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -73,13 +73,18 @@ jobs:
           echo $(pwd)/bin >> $GITHUB_PATH
           echo IN_DOCKER=1 >> .env
 
+
       - name: Start services
         run: |
           ci_localnet_up
           ci_run sccache --start-server
 
       - name: Initialize ecosystem
         run: |
+          ci_run git config --global --add safe.directory /usr/src/zksync
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts
+          ci_run git config --global --add safe.directory /usr/src/zksync/contracts
+          
           ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \
           --deploy-ecosystem --l1-rpc-url=http://reth:8545 \
           --server-db-url=postgres://postgres:notsecurepassword@postgres:5432 \

core/lib/contracts/src/lib.rs

@@ -39,8 +39,8 @@ const STATE_TRANSITION_CONTRACT_FILE: (&str, &str) = (
     "IStateTransitionManager.sol/IStateTransitionManager.json",
 );
 const ZKSYNC_HYPERCHAIN_CONTRACT_FILE: (&str, &str) = (
-    "state-transition/",
-    "chain-interfaces/IZkSyncHyperchain.sol/IZkSyncHyperchain.json",
+    "state-transition/chain-interfaces",
+    "IZkSyncHyperchain.sol/IZkSyncHyperchain.json",
 );
 const DIAMOND_INIT_CONTRACT_FILE: (&str, &str) = (
     "state-transition",

core/node/eth_sender/src/eth_tx_manager.rs

@@ -586,8 +586,19 @@ impl EthTxManager {
                 .await
                 .unwrap();
 
+            tracing::info!(
+                "Sending {} {operator_type:?} new transactions",
+                new_eth_tx.len()
+            );
             for tx in new_eth_tx {
-                let _ = self.send_eth_tx(storage, &tx, 0, current_block).await;
+                let result = self.send_eth_tx(storage, &tx, 0, current_block).await;
+                // If one of the transactions doesn't succeed, this means we should return
+                // as new transactions have increasing nonces, so they will also result in an error
+                // about gapped nonces
+                if result.is_err() {
+                    tracing::info!("Skipping sending rest of new transactions because of error");
+                    break;
+                }
             }
         }
     }

core/node/proof_data_handler/Cargo.toml

@@ -10,6 +10,7 @@ keywords.workspace = true
 categories.workspace = true
 
 [dependencies]
+vise.workspace = true
 zksync_config.workspace = true
 zksync_dal.workspace = true
 zksync_object_store.workspace = true

core/node/proof_data_handler/src/lib.rs

@@ -18,6 +18,7 @@ use zksync_types::commitment::L1BatchCommitmentMode;
 mod tests;
 
 mod errors;
+mod metrics;
 mod request_processor;
 mod tee_request_processor;
 

core/node/proof_data_handler/src/metrics.rs

@@ -0,0 +1,41 @@
+use vise::{Histogram, Metrics};
+use zksync_object_store::bincode;
+use zksync_prover_interface::inputs::WitnessInputData;
+
+const BYTES_IN_MEGABYTE: u64 = 1024 * 1024;
+
+#[derive(Debug, Metrics)]
+pub(super) struct ProofDataHandlerMetrics {
+    #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))]
+    pub vm_run_data_blob_size_in_mb: Histogram<u64>,
+    #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))]
+    pub merkle_paths_blob_size_in_mb: Histogram<u64>,
+    #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))]
+    pub eip_4844_blob_size_in_mb: Histogram<u64>,
+    #[metrics(buckets = vise::Buckets::exponential(1.0..=2_048.0, 2.0))]
+    pub total_blob_size_in_mb: Histogram<u64>,
+}
+
+impl ProofDataHandlerMetrics {
+    pub fn observe_blob_sizes(&self, blob: &WitnessInputData) {
+        let vm_run_data_blob_size_in_mb =
+            bincode::serialize(&blob.vm_run_data).unwrap().len() as u64 / BYTES_IN_MEGABYTE;
+        let merkle_paths_blob_size_in_mb =
+            bincode::serialize(&blob.merkle_paths).unwrap().len() as u64 / BYTES_IN_MEGABYTE;
+        let eip_4844_blob_size_in_mb =
+            bincode::serialize(&blob.eip_4844_blobs).unwrap().len() as u64 / BYTES_IN_MEGABYTE;
+        let total_blob_size_in_mb =
+            bincode::serialize(blob).unwrap().len() as u64 / BYTES_IN_MEGABYTE;
+
+        self.vm_run_data_blob_size_in_mb
+            .observe(vm_run_data_blob_size_in_mb);
+        self.merkle_paths_blob_size_in_mb
+            .observe(merkle_paths_blob_size_in_mb);
+        self.eip_4844_blob_size_in_mb
+            .observe(eip_4844_blob_size_in_mb);
+        self.total_blob_size_in_mb.observe(total_blob_size_in_mb);
+    }
+}
+
+#[vise::register]
+pub(super) static METRICS: vise::Global<ProofDataHandlerMetrics> = vise::Global::new();

core/node/proof_data_handler/src/request_processor.rs

@@ -20,7 +20,7 @@ use zksync_types::{
     L1BatchNumber, H256,
 };
 
-use crate::errors::RequestProcessorError;
+use crate::{errors::RequestProcessorError, metrics::METRICS};
 
 #[derive(Clone)]
 pub(crate) struct RequestProcessor {
@@ -147,6 +147,8 @@ impl RequestProcessor {
             },
         };
 
+        METRICS.observe_blob_sizes(&blob);
+
         let proof_gen_data = ProofGenerationData {
             l1_batch_number,
             witness_input_data: blob,

prover/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [16.0.0](https://github.com/matter-labs/zksync-era/compare/prover-v15.1.0...prover-v16.0.0) (2024-07-11)
+
+
+### ⚠ BREAKING CHANGES
+
+* **prover:** Bump prover protocol patch ([#2428](https://github.com/matter-labs/zksync-era/issues/2428))
+
+### Features
+
+* L1 batch signing (BFT-474) ([#2414](https://github.com/matter-labs/zksync-era/issues/2414)) ([ab699db](https://github.com/matter-labs/zksync-era/commit/ab699dbe8cffa8bd291d6054579061b47fd4aa0e))
+* **prover:** Bump prover protocol patch ([#2428](https://github.com/matter-labs/zksync-era/issues/2428)) ([1dffae9](https://github.com/matter-labs/zksync-era/commit/1dffae90d0d6a56434bb076135ac2a957ab20b83))
+
 ## [15.1.0](https://github.com/matter-labs/zksync-era/compare/prover-v15.0.0...prover-v15.1.0) (2024-07-10)
 
 

prover/prover_fri_types/src/lib.rs

@@ -27,7 +27,7 @@ pub mod queue;
 
 // THESE VALUES SHOULD BE UPDATED ON ANY PROTOCOL UPGRADE OF PROVERS
 pub const PROVER_PROTOCOL_VERSION: ProtocolVersionId = ProtocolVersionId::Version24;
-pub const PROVER_PROTOCOL_PATCH: VersionPatch = VersionPatch(1);
+pub const PROVER_PROTOCOL_PATCH: VersionPatch = VersionPatch(2);
 pub const PROVER_PROTOCOL_SEMANTIC_VERSION: ProtocolSemanticVersion = ProtocolSemanticVersion {
     minor: PROVER_PROTOCOL_VERSION,
     patch: PROVER_PROTOCOL_PATCH,

zk_toolbox/crates/common/src/cmd.rs

@@ -93,7 +93,13 @@ impl<'a> Cmd<'a> {
         let output = if global_config().verbose || self.force_run {
             logger::debug(format!("Running: {}", self.inner));
             logger::new_empty_line();
-            run_low_level_process_command(self.inner.into())?
+            let output = run_low_level_process_command(self.inner.into())?;
+            if let Ok(data) = String::from_utf8(output.stderr.clone()) {
+                if !data.is_empty() {
+                    logger::info(data)
+                }
+            }
+            output
         } else {
             // Command will be logged manually.
             self.inner.set_quiet(true);

zk_toolbox/crates/common/src/git.rs

@@ -0,0 +1,31 @@
+use std::path::PathBuf;
+
+use xshell::{cmd, Shell};
+
+use crate::cmd::Cmd;
+
+pub fn clone(
+    shell: &Shell,
+    path: PathBuf,
+    repository: &str,
+    name: &str,
+) -> anyhow::Result<PathBuf> {
+    let _dir = shell.push_dir(path);
+    Cmd::new(cmd!(
+        shell,
+        "git clone --recurse-submodules {repository} {name}"
+    ))
+    .run()?;
+    Ok(shell.current_dir().join(name))
+}
+
+pub fn submodule_update(shell: &Shell, link_to_code: PathBuf) -> anyhow::Result<()> {
+    let _dir_guard = shell.push_dir(link_to_code);
+    Cmd::new(cmd!(
+        shell,
+        "git submodule update --init --recursive
+"
+    ))
+    .run()?;
+    Ok(())
+}

zk_toolbox/crates/common/src/lib.rs

@@ -9,6 +9,7 @@ pub mod docker;
 pub mod ethereum;
 pub mod files;
 pub mod forge;
+pub mod git;
 pub mod server;
 pub mod wallets;
 

zk_toolbox/crates/config/src/ecosystem.rs

@@ -97,10 +97,12 @@ impl EcosystemConfig {
 
     pub fn from_file(shell: &Shell) -> Result<Self, EcosystemConfigFromFileError> {
         let Ok(path) = find_file(shell, shell.current_dir(), CONFIG_NAME) else {
-            return Err(EcosystemConfigFromFileError::NotExists);
+            return Err(EcosystemConfigFromFileError::NotExists {
+                path: shell.current_dir(),
+            });
         };
 
-        shell.change_dir(path);
+        shell.change_dir(&path);
 
         let ecosystem = match EcosystemConfig::read(shell, CONFIG_NAME) {
             Ok(mut config) => {
@@ -116,7 +118,7 @@ impl EcosystemConfig {
 
                 let current_dir = shell.current_dir();
                 let Some(parent) = current_dir.parent() else {
-                    return Err(EcosystemConfigFromFileError::NotExists);
+                    return Err(EcosystemConfigFromFileError::NotExists { path });
                 };
                 // Try to find ecosystem somewhere in parent directories
                 shell.change_dir(parent);
@@ -236,8 +238,9 @@ impl EcosystemConfig {
 /// Result of checking if the ecosystem exists.
 #[derive(Error, Debug)]
 pub enum EcosystemConfigFromFileError {
-    #[error("Ecosystem configuration not found")]
-    NotExists,
+    #[error("Ecosystem configuration not found (Could not find 'ZkStack.toml' in {path:?}: Make sure you have created an ecosystem & are in the new folder `cd path/to/ecosystem/name`)"
+    )]
+    NotExists { path: PathBuf },
     #[error("Invalid ecosystem configuration")]
     InvalidConfig { source: anyhow::Error },
 }

zk_toolbox/crates/zk_inception/src/commands/chain/init.rs

@@ -1,9 +1,8 @@
 use anyhow::Context;
 use common::{
-    cmd::Cmd,
     config::global_config,
     forge::{Forge, ForgeScriptArgs},
-    logger,
+    git, logger,
     spinner::Spinner,
 };
 use config::{
@@ -15,7 +14,7 @@ use config::{
     traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath},
     ChainConfig, ContractsConfig, EcosystemConfig,
 };
-use xshell::{cmd, Shell};
+use xshell::Shell;
 
 use crate::{
     accept_ownership::accept_admin,
@@ -26,9 +25,9 @@ use crate::{
         initialize_bridges,
     },
     messages::{
-        msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS,
-        MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR,
-        MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
+        msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED,
+        MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, MSG_REGISTERING_CHAIN_SPINNER,
+        MSG_SELECTED_CONFIG,
     },
     utils::forge::{check_the_balance, fill_forge_private_key},
 };
@@ -43,6 +42,7 @@ pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> {
 
     logger::note(MSG_SELECTED_CONFIG, logger::object_to_string(&chain_config));
     logger::info(msg_initializing_chain(""));
+    git::submodule_update(shell, config.link_to_code.clone())?;
 
     init(&mut args, shell, &config, &chain_config).await?;
 
@@ -57,7 +57,6 @@ pub async fn init(
     chain_config: &ChainConfig,
 ) -> anyhow::Result<()> {
     copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?;
-    build_l1_contracts(shell, ecosystem_config)?;
 
     let mut genesis_config = chain_config.get_genesis_config()?;
     genesis_config.update_from_chain_config(chain_config);
@@ -161,11 +160,3 @@ async fn register_chain(
     contracts.set_chain_contracts(&register_chain_output);
     Ok(())
 }
-
-fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> {
-    let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry());
-    let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS);
-    Cmd::new(cmd!(shell, "yarn build")).run()?;
-    spinner.finish();
-    Ok(())
-}

zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs

@@ -22,8 +22,10 @@ pub enum ChainCommands {
     /// Run server genesis
     Genesis(GenesisArgs),
     /// Initialize bridges on l2
+    #[command(alias = "bridge")]
     InitializeBridges(ForgeScriptArgs),
     /// Initialize bridges on l2
+    #[command(alias = "paymaster")]
     DeployPaymaster(ForgeScriptArgs),
 }
 

zk_toolbox/crates/zk_inception/src/commands/containers.rs

@@ -74,5 +74,8 @@ fn copy_dockerfile(shell: &Shell, link_to_code: PathBuf) -> anyhow::Result<()> {
 
     let data = docker_compose_text.replace(original_source, new_source);
     shell.write_file(DOCKER_COMPOSE_FILE, data)?;
+    // For some reasons our docker-compose sometimes required .env file while we are investigating this behaviour
+    // it's better to create file and don't make the life of customers harder
+    shell.write_file(".env", "")?;
     Ok(())
 }