feat(zk toolbox): External node support (#2287)

## What ❔

<!-- What are the changes this PR brings about? -->
<!-- Example: This PR adds a PR template to the repo. -->
<!-- (For bigger PRs adding more context is appreciated) -->

## Why ❔

<!-- Why are these changes done? What goal do they contribute to? What
are the principles behind them? -->
<!-- Example: PR templates ensure PR reviewers, observers, and future
iterators are in context about the evolution of repos. -->

## Checklist

<!-- Check your PR fulfills the following items. -->
<!-- For draft PRs check the boxes as you complete them. -->

- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.
- [ ] Spellcheck has been run via `zk spellcheck`.

---------

Signed-off-by: Danil <[email protected]>
Co-authored-by: Matías Ignacio González <[email protected]>

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -90,13 +90,30 @@ jobs:
 
       - name: Run server
         run: |
-          ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log &
+          ci_run zk_inception server --ignore-prerequisites &>server.log &
           ci_run sleep 5
 
       - name: Run integration tests
         run: |
           ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose
 
+
+      - name: Run external node server
+        run: |
+          ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545  
+          ci_run zk_inception external-node init --ignore-prerequisites
+          ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log &
+          ci_run sleep 5
+
+      - name: Run integration tests en
+        run: |
+          ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node
+
       - name: Show server.log logs
         if: always()
         run: ci_run cat server.log || true
+      - name: Show external_node.log logs
+        if: always()
+        run: ci_run cat external_node.log || true
+

bin/zkt

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+cd $(dirname $0)
+cd ../zk_toolbox
+
+cargo install --path ./crates/zk_inception --force
+cargo install --path ./crates/zk_supervisor --force

chains/era/ZkStack.yaml

@@ -4,6 +4,7 @@ chain_id: 271
 prover_version: NoProofs
 configs: ./chains/era/configs/
 rocks_db_path: ./chains/era/db/
+external_node_config_path: ./chains/era/configs/external_node
 l1_batch_commit_data_generator_mode: Rollup
 base_token:
   address: '0x0000000000000000000000000000000000000001'

core/bin/external_node/src/config/mod.rs

@@ -421,6 +421,9 @@ pub(crate) struct OptionalENConfig {
     #[serde(default = "OptionalENConfig::default_snapshots_recovery_postgres_max_concurrency")]
     pub snapshots_recovery_postgres_max_concurrency: NonZeroUsize,
 
+    #[serde(default)]
+    pub snapshot_recover_object_store: Option<ObjectStoreConfig>,
+
     /// Enables pruning of the historical node state (Postgres and Merkle tree). The node will retain
     /// recent state and will continuously remove (prune) old enough parts of the state in the background.
     #[serde(default)]
@@ -619,6 +622,10 @@ impl OptionalENConfig {
                 .as_ref()
                 .map(|a| a.enabled)
                 .unwrap_or_default(),
+            snapshot_recover_object_store: load_config!(
+                general_config.snapshot_recovery,
+                object_store
+            ),
             pruning_chunk_size: load_optional_config_or_default!(
                 general_config.pruning,
                 chunk_size,
@@ -798,9 +805,11 @@ impl OptionalENConfig {
     }
 
     fn from_env() -> anyhow::Result<Self> {
-        envy::prefixed("EN_")
+        let mut result: OptionalENConfig = envy::prefixed("EN_")
             .from_env()
-            .context("could not load external node config")
+            .context("could not load external node config")?;
+        result.snapshot_recover_object_store = snapshot_recovery_object_store_config().ok();
+        Ok(result)
     }
 
     pub fn polling_interval(&self) -> Duration {

core/bin/external_node/src/config/observability.rs

@@ -26,6 +26,8 @@ pub(crate) struct ObservabilityENConfig {
     /// Log format to use: either `plain` (default) or `json`.
     #[serde(default)]
     pub log_format: LogFormat,
+    // Log directives in format that is used in `RUST_LOG`
+    pub log_directives: Option<String>,
 }
 
 impl ObservabilityENConfig {
@@ -80,6 +82,9 @@ impl ObservabilityENConfig {
 
     pub fn build_observability(&self) -> anyhow::Result<zksync_vlog::ObservabilityGuard> {
         let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(self.log_format);
+        if let Some(log_directives) = self.log_directives.clone() {
+            builder = builder.with_log_directives(log_directives)
+        };
         // Some legacy deployments use `unset` as an equivalent of `None`.
         let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset");
         if let Some(sentry_url) = sentry_url {
@@ -100,7 +105,7 @@ impl ObservabilityENConfig {
     }
 
     pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result<Self> {
-        let (sentry_url, sentry_environment, log_format) =
+        let (sentry_url, sentry_environment, log_format, log_directives) =
             if let Some(observability) = general_config.observability.as_ref() {
                 (
                     observability.sentry_url.clone(),
@@ -109,9 +114,10 @@ impl ObservabilityENConfig {
                         .log_format
                         .parse()
                         .context("Invalid log format")?,
+                    observability.log_directives.clone(),
                 )
             } else {
-                (None, None, LogFormat::default())
+                (None, None, LogFormat::default(), None)
             };
         let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) =
             if let Some(prometheus) = general_config.prometheus_config.as_ref() {
@@ -130,6 +136,7 @@ impl ObservabilityENConfig {
             sentry_url,
             sentry_environment,
             log_format,
+            log_directives,
         })
     }
 }

core/bin/external_node/src/init.rs

@@ -3,6 +3,7 @@
 use std::time::Instant;
 
 use anyhow::Context as _;
+use zksync_config::ObjectStoreConfig;
 use zksync_dal::{ConnectionPool, Core, CoreDal};
 use zksync_health_check::AppHealthCheck;
 use zksync_node_sync::genesis::perform_genesis_if_needed;
@@ -12,12 +13,11 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask};
 use zksync_types::{L1BatchNumber, L2ChainId};
 use zksync_web3_decl::client::{DynClient, L2};
 
-use crate::config::snapshot_recovery_object_store_config;
-
 #[derive(Debug)]
 pub(crate) struct SnapshotRecoveryConfig {
     /// If not specified, the latest snapshot will be used.
     pub snapshot_l1_batch_override: Option<L1BatchNumber>,
+    pub object_store_config: Option<ObjectStoreConfig>,
 }
 
 #[derive(Debug)]
@@ -90,7 +90,9 @@ pub(crate) async fn ensure_storage_initialized(
             )?;
 
             tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk");
-            let object_store_config = snapshot_recovery_object_store_config()?;
+            let object_store_config = recovery_config.object_store_config.context(
+                "Snapshot object store must be presented if snapshot recovery is activated",
+            )?;
             let object_store = ObjectStoreFactory::new(object_store_config)
                 .create_store()
                 .await?;

core/bin/external_node/src/main.rs

@@ -971,6 +971,7 @@ async fn run_node(
             .snapshots_recovery_enabled
             .then_some(SnapshotRecoveryConfig {
                 snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch,
+                object_store_config: config.optional.snapshot_recover_object_store.clone(),
             });
     ensure_storage_initialized(
         connection_pool.clone(),

core/tests/ts-integration/src/env.ts

@@ -57,11 +57,18 @@ function getMainWalletPk(pathToHome: string, network: string): string {
  */
 async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironment> {
     const pathToHome = path.join(__dirname, '../../../..');
+    let nodeMode;
+    if (process.env.EXTERNAL_NODE == 'true') {
+        nodeMode = NodeMode.External;
+    } else {
+        nodeMode = NodeMode.Main;
+    }
     let ecosystem = loadEcosystem(pathToHome);
+    // Genesis file is common for both EN and Main node
+    let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml', NodeMode.Main);
 
-    let generalConfig = loadConfig(pathToHome, chain, 'general.yaml');
-    let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml');
-    let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml');
+    let generalConfig = loadConfig(pathToHome, chain, 'general.yaml', nodeMode);
+    let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml', nodeMode);
 
     const network = ecosystem.l1_network;
     let mainWalletPK = getMainWalletPk(pathToHome, network);
@@ -115,8 +122,6 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
     const l2ChainId = parseInt(genesisConfig.l2_chain_id);
     const l1BatchCommitDataGeneratorMode = genesisConfig.l1_batch_commit_data_generator_mode as DataAvailabityMode;
     let minimalL2GasPrice = generalConfig.state_keeper.minimal_l2_gas_price;
-    // TODO add support for en
-    let nodeMode = NodeMode.Main;
 
     const validationComputationalGasLimit = parseInt(generalConfig.state_keeper.validation_computational_gas_limit);
     // TODO set it properly
@@ -355,8 +360,13 @@ function loadEcosystem(pathToHome: string): any {
     );
 }
 
-function loadConfig(pathToHome: string, chainName: string, config: string): any {
-    const configPath = path.join(pathToHome, `/chains/${chainName}/configs/${config}`);
+function loadConfig(pathToHome: string, chainName: string, config: string, mode: NodeMode): any {
+    let configPath = path.join(pathToHome, `/chains/${chainName}/configs`);
+    let suffixPath = `${config}`;
+    if (mode == NodeMode.External) {
+        suffixPath = path.join('external_node', suffixPath);
+    }
+    configPath = path.join(configPath, suffixPath);
     if (!fs.existsSync(configPath)) {
         return [];
     }

etc/env/file_based/general.yaml

@@ -63,7 +63,7 @@ api:
     estimate_gas_scale_factor: 1.2
     estimate_gas_acceptable_overestimation: 1000
     max_tx_size: 1000000
-    api_namespaces: [ eth,net,web3,zks,pubsub ]
+    api_namespaces: [ eth,net,web3,zks,pubsub,debug ]
     max_response_body_size_overrides:
       - method: eth_getTransactionReceipt # no size specified, meaning no size limit
       - method: zks_getProof
@@ -319,7 +319,7 @@ prometheus:
 
 observability:
   log_format: plain
-  log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=debug,snapshots_creator=debug"
+  log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=error,snapshots_creator=debug"
   sentry:
     url: unset
     panic_interval: 1800
@@ -335,7 +335,7 @@ protective_reads_writer:
   first_processed_batch: 0
 
 snapshot_recovery:
-  enabled: true
+  enabled: false
   postgres:
     max_concurrency: 10
   tree:

zk_toolbox/crates/config/src/chain.rs

@@ -11,11 +11,12 @@ use xshell::Shell;
 
 use crate::{
     consts::{
-        CONFIG_NAME, CONTRACTS_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, SECRETS_FILE, WALLETS_FILE,
+        CONFIG_NAME, CONTRACTS_FILE, GENERAL_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY,
+        SECRETS_FILE, WALLETS_FILE,
     },
     create_localhost_wallets,
     traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath},
-    ContractsConfig, GenesisConfig, SecretsConfig, WalletsConfig,
+    ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig,
 };
 
 /// Chain configuration file. This file is created in the chain
@@ -30,6 +31,7 @@ pub struct ChainConfigInternal {
     pub prover_version: ProverMode,
     pub configs: PathBuf,
     pub rocks_db_path: PathBuf,
+    pub external_node_config_path: Option<PathBuf>,
     pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode,
     pub base_token: BaseToken,
     pub wallet_creation: WalletCreation,
@@ -47,6 +49,7 @@ pub struct ChainConfig {
     pub link_to_code: PathBuf,
     pub rocks_db_path: PathBuf,
     pub configs: PathBuf,
+    pub external_node_config_path: Option<PathBuf>,
     pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode,
     pub base_token: BaseToken,
     pub wallet_creation: WalletCreation,
@@ -71,6 +74,10 @@ impl ChainConfig {
         GenesisConfig::read(self.get_shell(), self.configs.join(GENESIS_FILE))
     }
 
+    pub fn get_general_config(&self) -> anyhow::Result<GeneralConfig> {
+        GeneralConfig::read(self.get_shell(), self.configs.join(GENERAL_FILE))
+    }
+
     pub fn get_wallets_config(&self) -> anyhow::Result<WalletsConfig> {
         let path = self.configs.join(WALLETS_FILE);
         if let Ok(wallets) = WalletsConfig::read(self.get_shell(), &path) {
@@ -100,7 +107,7 @@ impl ChainConfig {
         config.save(shell, path)
     }
 
-    pub fn save_with_base_path(&self, shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<()> {
+    pub fn save_with_base_path(self, shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<()> {
         let config = self.get_internal();
         config.save_with_base_path(shell, path)
     }
@@ -113,6 +120,7 @@ impl ChainConfig {
             prover_version: self.prover_version,
             configs: self.configs.clone(),
             rocks_db_path: self.rocks_db_path.clone(),
+            external_node_config_path: self.external_node_config_path.clone(),
             l1_batch_commit_data_generator_mode: self.l1_batch_commit_data_generator_mode,
             base_token: self.base_token.clone(),
             wallet_creation: self.wallet_creation,

zk_toolbox/crates/config/src/consts.rs

@@ -11,6 +11,8 @@ pub(crate) const GENERAL_FILE: &str = "general.yaml";
 /// Name of the genesis config file
 pub(crate) const GENESIS_FILE: &str = "genesis.yaml";
 
+// Name of external node specific config
+pub(crate) const EN_CONFIG_FILE: &str = "external_node.yaml";
 pub(crate) const ERC20_CONFIGS_FILE: &str = "erc20.yaml";
 /// Name of the initial deployments config file
 pub(crate) const INITIAL_DEPLOYMENT_FILE: &str = "initial_deployments.yaml";

zk_toolbox/crates/config/src/contracts.rs

@@ -3,7 +3,11 @@ use serde::{Deserialize, Serialize};
 
 use crate::{
     consts::CONTRACTS_FILE,
-    forge_interface::deploy_ecosystem::output::DeployL1Output,
+    forge_interface::{
+        deploy_ecosystem::output::DeployL1Output,
+        initialize_bridges::output::InitializeBridgeOutput,
+        register_chain::output::RegisterChainOutput,
+    },
     traits::{FileConfig, FileConfigWithDefaultName},
 };
 
@@ -64,6 +68,20 @@ impl ContractsConfig {
             .diamond_cut_data
             .clone_from(&deploy_l1_output.contracts_config.diamond_cut_data);
     }
+
+    pub fn set_chain_contracts(&mut self, register_chain_output: &RegisterChainOutput) {
+        self.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr;
+        self.l1.governance_addr = register_chain_output.governance_addr;
+    }
+
+    pub fn set_l2_shared_bridge(
+        &mut self,
+        initialize_bridges_output: &InitializeBridgeOutput,
+    ) -> anyhow::Result<()> {
+        self.bridges.shared.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy);
+        self.bridges.erc20.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy);
+        Ok(())
+    }
 }
 
 impl FileConfigWithDefaultName for ContractsConfig {

zk_toolbox/crates/config/src/ecosystem.rs

@@ -120,6 +120,7 @@ impl EcosystemConfig {
             chain_id: config.chain_id,
             prover_version: config.prover_version,
             configs: config.configs,
+            external_node_config_path: config.external_node_config_path,
             l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode,
             l1_network: self.l1_network,
             link_to_code: self

zk_toolbox/crates/config/src/external_node.rs

@@ -0,0 +1,23 @@
+use std::num::NonZeroUsize;
+
+use serde::{Deserialize, Serialize};
+use types::{ChainId, L1BatchCommitDataGeneratorMode};
+
+use crate::{consts::EN_CONFIG_FILE, traits::FileConfigWithDefaultName};
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)]
+pub struct ENConfig {
+    // Genesis
+    pub l2_chain_id: ChainId,
+    pub l1_chain_id: u32,
+    pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode,
+
+    // Main node configuration
+    pub main_node_url: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub main_node_rate_limit_rps: Option<NonZeroUsize>,
+}
+
+impl FileConfigWithDefaultName for ENConfig {
+    const FILE_NAME: &'static str = EN_CONFIG_FILE;
+}