diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 66e54bfa98a4..83ec7d1f5dc3 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -90,13 +90,30 @@ jobs: - name: Run server run: | - ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log & + ci_run zk_inception server --ignore-prerequisites &>server.log & ci_run sleep 5 - name: Run integration tests run: | ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose + + - name: Run external node server + run: | + ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \ + --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 + ci_run zk_inception external-node init --ignore-prerequisites + ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log & + ci_run sleep 5 + + - name: Run integration tests en + run: | + ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node + - name: Show server.log logs if: always() run: ci_run cat server.log || true + - name: Show external_node.log logs + if: always() + run: ci_run cat external_node.log || true + diff --git a/bin/zkt b/bin/zkt new file mode 100755 index 000000000000..337ad5d73953 --- /dev/null +++ b/bin/zkt @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +cd $(dirname $0) +cd ../zk_toolbox + +cargo install --path ./crates/zk_inception --force +cargo install --path ./crates/zk_supervisor --force diff --git a/chains/era/ZkStack.yaml b/chains/era/ZkStack.yaml index 17b307cac4f6..8dbd49c02c67 100644 --- a/chains/era/ZkStack.yaml +++ b/chains/era/ZkStack.yaml @@ -4,6 +4,7 @@ chain_id: 271 prover_version: NoProofs configs: ./chains/era/configs/ rocks_db_path: ./chains/era/db/ +external_node_config_path: ./chains/era/configs/external_node l1_batch_commit_data_generator_mode: Rollup base_token: address: '0x0000000000000000000000000000000000000001' diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 35750cfa4e7d..b5b041a1fc6e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -421,6 +421,9 @@ pub(crate) struct OptionalENConfig { #[serde(default = "OptionalENConfig::default_snapshots_recovery_postgres_max_concurrency")] pub snapshots_recovery_postgres_max_concurrency: NonZeroUsize, + #[serde(default)] + pub snapshot_recover_object_store: Option, + /// Enables pruning of the historical node state (Postgres and Merkle tree). The node will retain /// recent state and will continuously remove (prune) old enough parts of the state in the background. #[serde(default)] @@ -619,6 +622,10 @@ impl OptionalENConfig { .as_ref() .map(|a| a.enabled) .unwrap_or_default(), + snapshot_recover_object_store: load_config!( + general_config.snapshot_recovery, + object_store + ), pruning_chunk_size: load_optional_config_or_default!( general_config.pruning, chunk_size, @@ -798,9 +805,11 @@ impl OptionalENConfig { } fn from_env() -> anyhow::Result { - envy::prefixed("EN_") + let mut result: OptionalENConfig = envy::prefixed("EN_") .from_env() - .context("could not load external node config") + .context("could not load external node config")?; + result.snapshot_recover_object_store = snapshot_recovery_object_store_config().ok(); + Ok(result) } pub fn polling_interval(&self) -> Duration { diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index 39b86b8f0452..4dc310ee26c2 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -26,6 +26,8 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, + // Log directives in format that is used in `RUST_LOG` + pub log_directives: Option, } impl ObservabilityENConfig { @@ -80,6 +82,9 @@ impl ObservabilityENConfig { pub fn build_observability(&self) -> anyhow::Result { let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(self.log_format); + if let Some(log_directives) = self.log_directives.clone() { + builder = builder.with_log_directives(log_directives) + }; // Some legacy deployments use `unset` as an equivalent of `None`. let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset"); if let Some(sentry_url) = sentry_url { @@ -100,7 +105,7 @@ impl ObservabilityENConfig { } pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { - let (sentry_url, sentry_environment, log_format) = + let (sentry_url, sentry_environment, log_format, log_directives) = if let Some(observability) = general_config.observability.as_ref() { ( observability.sentry_url.clone(), @@ -109,9 +114,10 @@ impl ObservabilityENConfig { .log_format .parse() .context("Invalid log format")?, + observability.log_directives.clone(), ) } else { - (None, None, LogFormat::default()) + (None, None, LogFormat::default(), None) }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = if let Some(prometheus) = general_config.prometheus_config.as_ref() { @@ -130,6 +136,7 @@ impl ObservabilityENConfig { sentry_url, sentry_environment, log_format, + log_directives, }) } } diff --git a/core/bin/external_node/src/init.rs b/core/bin/external_node/src/init.rs index a9ee796194cc..ddf83a1f5581 100644 --- a/core/bin/external_node/src/init.rs +++ b/core/bin/external_node/src/init.rs @@ -3,6 +3,7 @@ use std::time::Instant; use anyhow::Context as _; +use zksync_config::ObjectStoreConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_health_check::AppHealthCheck; use zksync_node_sync::genesis::perform_genesis_if_needed; @@ -12,12 +13,11 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask}; use zksync_types::{L1BatchNumber, L2ChainId}; use zksync_web3_decl::client::{DynClient, L2}; -use crate::config::snapshot_recovery_object_store_config; - #[derive(Debug)] pub(crate) struct SnapshotRecoveryConfig { /// If not specified, the latest snapshot will be used. pub snapshot_l1_batch_override: Option, + pub object_store_config: Option, } #[derive(Debug)] @@ -90,7 +90,9 @@ pub(crate) async fn ensure_storage_initialized( )?; tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk"); - let object_store_config = snapshot_recovery_object_store_config()?; + let object_store_config = recovery_config.object_store_config.context( + "Snapshot object store must be presented if snapshot recovery is activated", + )?; let object_store = ObjectStoreFactory::new(object_store_config) .create_store() .await?; diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index c54bdc1dab19..0b3854b03c05 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -971,6 +971,7 @@ async fn run_node( .snapshots_recovery_enabled .then_some(SnapshotRecoveryConfig { snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch, + object_store_config: config.optional.snapshot_recover_object_store.clone(), }); ensure_storage_initialized( connection_pool.clone(), diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index c440e6b08ea6..ca97363fb4e2 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -57,11 +57,18 @@ function getMainWalletPk(pathToHome: string, network: string): string { */ async function loadTestEnvironmentFromFile(chain: string): Promise { const pathToHome = path.join(__dirname, '../../../..'); + let nodeMode; + if (process.env.EXTERNAL_NODE == 'true') { + nodeMode = NodeMode.External; + } else { + nodeMode = NodeMode.Main; + } let ecosystem = loadEcosystem(pathToHome); + // Genesis file is common for both EN and Main node + let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml', NodeMode.Main); - let generalConfig = loadConfig(pathToHome, chain, 'general.yaml'); - let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml'); - let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml'); + let generalConfig = loadConfig(pathToHome, chain, 'general.yaml', nodeMode); + let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml', nodeMode); const network = ecosystem.l1_network; let mainWalletPK = getMainWalletPk(pathToHome, network); @@ -115,8 +122,6 @@ async function loadTestEnvironmentFromFile(chain: string): Promise, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -47,6 +49,7 @@ pub struct ChainConfig { pub link_to_code: PathBuf, pub rocks_db_path: PathBuf, pub configs: PathBuf, + pub external_node_config_path: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -71,6 +74,10 @@ impl ChainConfig { GenesisConfig::read(self.get_shell(), self.configs.join(GENESIS_FILE)) } + pub fn get_general_config(&self) -> anyhow::Result { + GeneralConfig::read(self.get_shell(), self.configs.join(GENERAL_FILE)) + } + pub fn get_wallets_config(&self) -> anyhow::Result { let path = self.configs.join(WALLETS_FILE); if let Ok(wallets) = WalletsConfig::read(self.get_shell(), &path) { @@ -100,7 +107,7 @@ impl ChainConfig { config.save(shell, path) } - pub fn save_with_base_path(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + pub fn save_with_base_path(self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { let config = self.get_internal(); config.save_with_base_path(shell, path) } @@ -113,6 +120,7 @@ impl ChainConfig { prover_version: self.prover_version, configs: self.configs.clone(), rocks_db_path: self.rocks_db_path.clone(), + external_node_config_path: self.external_node_config_path.clone(), l1_batch_commit_data_generator_mode: self.l1_batch_commit_data_generator_mode, base_token: self.base_token.clone(), wallet_creation: self.wallet_creation, diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 9141d044af94..a00274fb35f3 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -11,6 +11,8 @@ pub(crate) const GENERAL_FILE: &str = "general.yaml"; /// Name of the genesis config file pub(crate) const GENESIS_FILE: &str = "genesis.yaml"; +// Name of external node specific config +pub(crate) const EN_CONFIG_FILE: &str = "external_node.yaml"; pub(crate) const ERC20_CONFIGS_FILE: &str = "erc20.yaml"; /// Name of the initial deployments config file pub(crate) const INITIAL_DEPLOYMENT_FILE: &str = "initial_deployments.yaml"; diff --git a/zk_toolbox/crates/config/src/contracts.rs b/zk_toolbox/crates/config/src/contracts.rs index b86b9b0f2958..a847c8a4cc93 100644 --- a/zk_toolbox/crates/config/src/contracts.rs +++ b/zk_toolbox/crates/config/src/contracts.rs @@ -3,7 +3,11 @@ use serde::{Deserialize, Serialize}; use crate::{ consts::CONTRACTS_FILE, - forge_interface::deploy_ecosystem::output::DeployL1Output, + forge_interface::{ + deploy_ecosystem::output::DeployL1Output, + initialize_bridges::output::InitializeBridgeOutput, + register_chain::output::RegisterChainOutput, + }, traits::{FileConfig, FileConfigWithDefaultName}, }; @@ -64,6 +68,20 @@ impl ContractsConfig { .diamond_cut_data .clone_from(&deploy_l1_output.contracts_config.diamond_cut_data); } + + pub fn set_chain_contracts(&mut self, register_chain_output: &RegisterChainOutput) { + self.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; + self.l1.governance_addr = register_chain_output.governance_addr; + } + + pub fn set_l2_shared_bridge( + &mut self, + initialize_bridges_output: &InitializeBridgeOutput, + ) -> anyhow::Result<()> { + self.bridges.shared.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + self.bridges.erc20.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + Ok(()) + } } impl FileConfigWithDefaultName for ContractsConfig { diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 1557ab21646f..08708ebb0b61 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -120,6 +120,7 @@ impl EcosystemConfig { chain_id: config.chain_id, prover_version: config.prover_version, configs: config.configs, + external_node_config_path: config.external_node_config_path, l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode, l1_network: self.l1_network, link_to_code: self diff --git a/zk_toolbox/crates/config/src/external_node.rs b/zk_toolbox/crates/config/src/external_node.rs new file mode 100644 index 000000000000..87acb15e4d8c --- /dev/null +++ b/zk_toolbox/crates/config/src/external_node.rs @@ -0,0 +1,23 @@ +use std::num::NonZeroUsize; + +use serde::{Deserialize, Serialize}; +use types::{ChainId, L1BatchCommitDataGeneratorMode}; + +use crate::{consts::EN_CONFIG_FILE, traits::FileConfigWithDefaultName}; + +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] +pub struct ENConfig { + // Genesis + pub l2_chain_id: ChainId, + pub l1_chain_id: u32, + pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, + + // Main node configuration + pub main_node_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub main_node_rate_limit_rps: Option, +} + +impl FileConfigWithDefaultName for ENConfig { + const FILE_NAME: &'static str = EN_CONFIG_FILE; +} diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 058f23bf1b5d..e1f3655d2200 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -1,17 +1,68 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; +use url::Url; use crate::{consts::GENERAL_FILE, traits::FileConfigWithDefaultName}; +pub struct RocksDbs { + pub state_keeper: PathBuf, + pub merkle_tree: PathBuf, +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GeneralConfig { pub db: RocksDBConfig, pub eth: EthConfig, + pub api: ApiConfig, #[serde(flatten)] pub other: serde_json::Value, } +impl GeneralConfig { + pub fn set_rocks_db_config(&mut self, rocks_dbs: RocksDbs) -> anyhow::Result<()> { + self.db.state_keeper_db_path = rocks_dbs.state_keeper; + self.db.merkle_tree.path = rocks_dbs.merkle_tree; + Ok(()) + } + + pub fn ports_config(&self) -> PortsConfig { + PortsConfig { + web3_json_rpc_http_port: self.api.web3_json_rpc.http_port, + web3_json_rpc_ws_port: self.api.web3_json_rpc.ws_port, + healthcheck_port: self.api.healthcheck.port, + merkle_tree_port: self.api.merkle_tree.port, + prometheus_listener_port: self.api.prometheus.listener_port, + } + } + + pub fn update_ports(&mut self, ports_config: &PortsConfig) -> anyhow::Result<()> { + self.api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; + update_port_in_url( + &mut self.api.web3_json_rpc.http_url, + ports_config.web3_json_rpc_http_port, + )?; + self.api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port; + update_port_in_url( + &mut self.api.web3_json_rpc.ws_url, + ports_config.web3_json_rpc_ws_port, + )?; + self.api.healthcheck.port = ports_config.healthcheck_port; + self.api.merkle_tree.port = ports_config.merkle_tree_port; + self.api.prometheus.listener_port = ports_config.prometheus_listener_port; + Ok(()) + } +} + +fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> { + let mut http_url_url = Url::parse(&http_url)?; + if let Err(()) = http_url_url.set_port(Some(port)) { + anyhow::bail!("Wrong url, setting port is impossible"); + } + *http_url = http_url_url.as_str().to_string(); + Ok(()) +} + impl FileConfigWithDefaultName for GeneralConfig { const FILE_NAME: &'static str = GENERAL_FILE; } @@ -45,3 +96,88 @@ pub struct EthSender { #[serde(flatten)] pub other: serde_json::Value, } + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct ApiConfig { + /// Configuration options for the Web3 JSON RPC servers. + pub web3_json_rpc: Web3JsonRpcConfig, + /// Configuration options for the Prometheus exporter. + pub prometheus: PrometheusConfig, + /// Configuration options for the Health check. + pub healthcheck: HealthCheckConfig, + /// Configuration options for Merkle tree API. + pub merkle_tree: MerkleTreeApiConfig, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct Web3JsonRpcConfig { + /// Port to which the HTTP RPC server is listening. + pub http_port: u16, + /// URL to access HTTP RPC server. + pub http_url: String, + /// Port to which the WebSocket RPC server is listening. + pub ws_port: u16, + /// URL to access WebSocket RPC server. + pub ws_url: String, + /// Max possible limit of entities to be requested once. + pub req_entities_limit: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct PrometheusConfig { + /// Port to which the Prometheus exporter server is listening. + pub listener_port: u16, + /// URL of the push gateway. + pub pushgateway_url: String, + /// Push interval in ms. + pub push_interval_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct HealthCheckConfig { + /// Port to which the REST server is listening. + pub port: u16, + /// Time limit in milliseconds to mark a health check as slow and log the corresponding warning. + /// If not specified, the default value in the health check crate will be used. + pub slow_time_limit_ms: Option, + /// Time limit in milliseconds to abort a health check and return "not ready" status for the corresponding component. + /// If not specified, the default value in the health check crate will be used. + pub hard_time_limit_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +/// Configuration for the Merkle tree API. +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct MerkleTreeApiConfig { + /// Port to bind the Merkle tree API server to. + pub port: u16, + #[serde(flatten)] + pub other: serde_json::Value, +} + +pub struct PortsConfig { + pub web3_json_rpc_http_port: u16, + pub web3_json_rpc_ws_port: u16, + pub healthcheck_port: u16, + pub merkle_tree_port: u16, + pub prometheus_listener_port: u16, +} + +impl PortsConfig { + pub fn next_empty_ports_config(&self) -> PortsConfig { + Self { + web3_json_rpc_http_port: self.web3_json_rpc_http_port + 100, + web3_json_rpc_ws_port: self.web3_json_rpc_ws_port + 100, + healthcheck_port: self.healthcheck_port + 100, + merkle_tree_port: self.merkle_tree_port + 100, + prometheus_listener_port: self.prometheus_listener_port + 100, + } + } +} diff --git a/zk_toolbox/crates/config/src/genesis.rs b/zk_toolbox/crates/config/src/genesis.rs index 4e3d931ea0f0..e666931870a8 100644 --- a/zk_toolbox/crates/config/src/genesis.rs +++ b/zk_toolbox/crates/config/src/genesis.rs @@ -2,7 +2,7 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; use types::{ChainId, L1BatchCommitDataGeneratorMode, ProtocolSemanticVersion}; -use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName}; +use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName, ChainConfig}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GenesisConfig { @@ -21,6 +21,14 @@ pub struct GenesisConfig { pub other: serde_json::Value, } +impl GenesisConfig { + pub fn update_from_chain_config(&mut self, config: &ChainConfig) { + self.l2_chain_id = config.chain_id; + self.l1_chain_id = config.l1_network.chain_id(); + self.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); + } +} + impl FileConfigWithDefaultName for GenesisConfig { const FILE_NAME: &'static str = GENESIS_FILE; } diff --git a/zk_toolbox/crates/config/src/lib.rs b/zk_toolbox/crates/config/src/lib.rs index 8e40da7bf6bd..a80a2b6fe5de 100644 --- a/zk_toolbox/crates/config/src/lib.rs +++ b/zk_toolbox/crates/config/src/lib.rs @@ -1,3 +1,15 @@ +pub use chain::*; +pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; +pub use contracts::*; +pub use ecosystem::*; +pub use file_config::*; +pub use general::*; +pub use genesis::*; +pub use manipulations::*; +pub use secrets::*; +pub use wallet_creation::*; +pub use wallets::*; + mod chain; mod consts; mod contracts; @@ -10,17 +22,6 @@ mod secrets; mod wallet_creation; mod wallets; +pub mod external_node; pub mod forge_interface; pub mod traits; - -pub use chain::*; -pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; -pub use contracts::*; -pub use ecosystem::*; -pub use file_config::*; -pub use general::*; -pub use genesis::*; -pub use manipulations::*; -pub use secrets::*; -pub use wallet_creation::*; -pub use wallets::*; diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index ebacc5d437cb..98a9be6ffe61 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -1,3 +1,4 @@ +use common::db::DatabaseConfig; use serde::{Deserialize, Serialize}; use url::Url; @@ -6,7 +7,8 @@ use crate::{consts::SECRETS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabaseSecrets { pub server_url: Url, - pub prover_url: Url, + #[serde(skip_serializing_if = "Option::is_none")] + pub prover_url: Option, #[serde(flatten)] pub other: serde_json::Value, } @@ -26,6 +28,21 @@ pub struct SecretsConfig { pub other: serde_json::Value, } +impl SecretsConfig { + pub fn set_databases( + &mut self, + server_db_config: &DatabaseConfig, + prover_db_config: &DatabaseConfig, + ) { + self.database.server_url = server_db_config.full_url(); + self.database.prover_url = Some(prover_db_config.full_url()); + } + + pub fn set_l1_rpc_url(&mut self, l1_rpc_url: String) { + self.l1.l1_rpc_url = l1_rpc_url; + } +} + impl FileConfigWithDefaultName for SecretsConfig { const FILE_NAME: &'static str = SECRETS_FILE; } diff --git a/zk_toolbox/crates/config/src/traits.rs b/zk_toolbox/crates/config/src/traits.rs index 85c73e99f99b..79ae3a187a8b 100644 --- a/zk_toolbox/crates/config/src/traits.rs +++ b/zk_toolbox/crates/config/src/traits.rs @@ -18,11 +18,17 @@ pub trait FileConfigWithDefaultName { } impl FileConfig for T where T: FileConfigWithDefaultName {} + impl ReadConfig for T where T: FileConfig + Clone + DeserializeOwned {} + impl SaveConfig for T where T: FileConfig + Serialize {} + impl SaveConfigWithComment for T where T: FileConfig + Serialize {} + impl ReadConfigWithBasePath for T where T: FileConfigWithDefaultName + Clone + DeserializeOwned {} + impl SaveConfigWithBasePath for T where T: FileConfigWithDefaultName + Serialize {} + impl SaveConfigWithCommentAndBasePath for T where T: FileConfigWithDefaultName + Serialize {} /// Reads a config file from a given path, correctly parsing file extension. diff --git a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs index 830da513d4f0..179cb696ac3d 100644 --- a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs +++ b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs @@ -13,8 +13,8 @@ use ethers::types::{Address, H256}; use xshell::Shell; use crate::{ - forge_utils::{check_the_balance, fill_forge_private_key}, messages::MSG_ACCEPTING_GOVERNANCE_SPINNER, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn accept_admin( diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs index bf1457ba92c6..7b21015691b9 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs @@ -1,3 +1,3 @@ -mod run_server; - pub use run_server::*; + +mod run_server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs index 1ec211c25f6d..74bafd6ce5ef 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs @@ -13,5 +13,5 @@ pub struct RunServerArgs { pub genesis: bool, #[clap(long, short)] #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] - additional_args: Vec, + pub additional_args: Vec, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs index 0b0529ea5139..483b78e9b267 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs @@ -9,8 +9,8 @@ use crate::{ defaults::{generate_db_names, DBNames, DATABASE_PROVER_URL, DATABASE_SERVER_URL}, messages::{ msg_prover_db_name_prompt, msg_prover_db_url_prompt, msg_server_db_name_prompt, - msg_server_db_url_prompt, MSG_GENESIS_USE_DEFAULT_HELP, MSG_PROVER_DB_NAME_HELP, - MSG_PROVER_DB_URL_HELP, MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, + msg_server_db_url_prompt, MSG_PROVER_DB_NAME_HELP, MSG_PROVER_DB_URL_HELP, + MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, MSG_USE_DEFAULT_DATABASES_HELP, }, }; @@ -24,7 +24,7 @@ pub struct GenesisArgs { pub prover_db_url: Option, #[clap(long, help = MSG_PROVER_DB_NAME_HELP)] pub prover_db_name: Option, - #[clap(long, short, help = MSG_GENESIS_USE_DEFAULT_HELP)] + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] pub use_default: bool, #[clap(long, short, action)] pub dont_drop: bool, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs index f915a3b8d6f6..dc8f408db3b3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs @@ -68,6 +68,7 @@ pub(crate) fn create_chain_inner( link_to_code: ecosystem_config.link_to_code.clone(), rocks_db_path: ecosystem_config.get_chain_rocks_db_path(&default_chain_name), configs: chain_configs_path.clone(), + external_node_config_path: None, l1_batch_commit_data_generator_mode: args.l1_batch_commit_data_generator_mode, base_token: args.base_token, wallet_creation: args.wallet_creation, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs index fe8dcdc562b2..4f82a92c2edc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs @@ -9,15 +9,14 @@ use config::{ paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, }; use xshell::Shell; use crate::{ - config_manipulations::update_paymaster, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { @@ -26,12 +25,15 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_config = ecosystem_config .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; - deploy_paymaster(shell, &chain_config, args).await + let mut contracts = chain_config.get_contracts_config()?; + deploy_paymaster(shell, &chain_config, &mut contracts, args).await?; + contracts.save_with_base_path(shell, chain_config.configs) } pub async fn deploy_paymaster( shell: &Shell, chain_config: &ChainConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { let input = DeployPaymasterInput::new(chain_config)?; @@ -63,6 +65,6 @@ pub async fn deploy_paymaster( DEPLOY_PAYMASTER_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_paymaster(shell, chain_config, &output)?; + contracts_config.l2.testnet_paymaster_addr = output.paymaster; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 8c4edc88290d..554f9c2cf940 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -7,26 +7,25 @@ use common::{ logger, spinner::Spinner, }; -use config::{ChainConfig, EcosystemConfig}; +use config::{traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig}; +use types::ProverMode; use xshell::Shell; use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, - config_manipulations::{update_database_secrets, update_general_config}, + consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, MSG_INITIALIZING_PROVER_DATABASE, - MSG_INITIALIZING_SERVER_DATABASE, MSG_SELECTED_CONFIG, MSG_STARTING_GENESIS, - MSG_STARTING_GENESIS_SPINNER, + MSG_INITIALIZING_SERVER_DATABASE, MSG_RECREATE_ROCKS_DB_ERRROR, MSG_SELECTED_CONFIG, + MSG_STARTING_GENESIS, MSG_STARTING_GENESIS_SPINNER, }, server::{RunServer, ServerMode}, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; -const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; -const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; - pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -46,12 +45,20 @@ pub async fn genesis( shell: &Shell, config: &ChainConfig, ) -> anyhow::Result<()> { - // Clean the rocksdb - shell.remove_path(&config.rocks_db_path)?; shell.create_dir(&config.rocks_db_path)?; - update_general_config(shell, config)?; - update_database_secrets(shell, config, &args.server_db, &args.prover_db)?; + let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) + .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; + let mut general = config.get_general_config()?; + general.set_rocks_db_config(rocks_db)?; + if config.prover_version != ProverMode::NoProofs { + general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); + } + general.save_with_base_path(shell, &config.configs)?; + + let mut secrets = config.get_secrets_config()?; + secrets.set_databases(&args.server_db, &args.prover_db); + secrets.save_with_base_path(&shell, &config.configs)?; logger::note( MSG_SELECTED_CONFIG, @@ -128,5 +135,5 @@ async fn initialize_databases( fn run_server_genesis(chain_config: &ChainConfig, shell: &Shell) -> anyhow::Result<()> { let server = RunServer::new(None, chain_config); - server.run(shell, ServerMode::Genesis) + server.run(shell, ServerMode::Genesis, vec![]) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 0c9ac8743eee..9660e30da15f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -1,5 +1,6 @@ use anyhow::Context; use common::{ + cmd::Cmd, config::global_config, forge::{Forge, ForgeScriptArgs}, logger, @@ -11,24 +12,25 @@ use config::{ register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, script_params::REGISTER_CHAIN_SCRIPT_PARAMS, }, - traits::{ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath}, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, ChainConfig, ContractsConfig, EcosystemConfig, }; -use xshell::Shell; +use xshell::{cmd, Shell}; -use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ - args::init::InitArgs, deploy_paymaster, genesis::genesis, initialize_bridges, + args::init::{InitArgs, InitArgsFinal}, + deploy_paymaster, + genesis::genesis, + initialize_bridges, }, - config_manipulations::{update_genesis, update_l1_contracts, update_l1_rpc_url_secret}, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ - msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, - MSG_CHAIN_NOT_FOUND_ERR, MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, + msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, + MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { @@ -55,24 +57,32 @@ pub async fn init( chain_config: &ChainConfig, ) -> anyhow::Result<()> { copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; + build_l1_contracts(shell, ecosystem_config)?; + + let mut genesis_config = chain_config.get_genesis_config()?; + genesis_config.update_from_chain_config(&chain_config); + genesis_config.save_with_base_path(shell, &chain_config.configs)?; - update_genesis(shell, chain_config)?; - update_l1_rpc_url_secret(shell, chain_config, init_args.l1_rpc_url.clone())?; - let mut contracts_config = - ContractsConfig::read_with_base_path(shell, &ecosystem_config.config)?; - contracts_config.l1.base_token_addr = chain_config.base_token.address; // Copy ecosystem contracts + let mut contracts_config = ecosystem_config.get_contracts_config()?; + contracts_config.l1.base_token_addr = chain_config.base_token.address; contracts_config.save_with_base_path(shell, &chain_config.configs)?; + let mut secrets = chain_config.get_secrets_config()?; + secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); + secrets.save_with_base_path(shell, &chain_config.configs)?; + let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); - contracts_config = register_chain( + register_chain( shell, init_args.forge_args.clone(), ecosystem_config, chain_config, + &mut contracts_config, init_args.l1_rpc_url.clone(), ) .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); accept_admin( @@ -91,13 +101,21 @@ pub async fn init( shell, chain_config, ecosystem_config, + &mut contracts_config, init_args.forge_args.clone(), ) .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; if init_args.deploy_paymaster { - deploy_paymaster::deploy_paymaster(shell, chain_config, init_args.forge_args.clone()) - .await?; + deploy_paymaster::deploy_paymaster( + shell, + chain_config, + &mut contracts_config, + init_args.forge_args.clone(), + ) + .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; } genesis(init_args.genesis_args.clone(), shell, chain_config) @@ -112,13 +130,11 @@ async fn register_chain( forge_args: ForgeScriptArgs, config: &EcosystemConfig, chain_config: &ChainConfig, + contracts: &mut ContractsConfig, l1_rpc_url: String, -) -> anyhow::Result { +) -> anyhow::Result<()> { let deploy_config_path = REGISTER_CHAIN_SCRIPT_PARAMS.input(&config.link_to_code); - let contracts = config - .get_contracts_config() - .context(MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR)?; let deploy_config = RegisterChainL1Config::new(chain_config, &contracts)?; deploy_config.save(shell, deploy_config_path)?; @@ -136,5 +152,14 @@ async fn register_chain( shell, REGISTER_CHAIN_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l1_contracts(shell, chain_config, ®ister_chain_output) + contracts.set_chain_contracts(®ister_chain_output); + Ok(()) +} + +fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); + let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); + Cmd::new(cmd!(shell, "yarn build")).run()?; + spinner.finish(); + Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs index 4a81a2b26f1b..2fab4f8ae6d4 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs @@ -12,15 +12,14 @@ use config::{ initialize_bridges::{input::InitializeBridgeInput, output::InitializeBridgeOutput}, script_params::INITIALIZE_BRIDGES_SCRIPT_PARAMS, }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, }; use xshell::{cmd, Shell}; use crate::{ - config_manipulations::update_l2_shared_bridge, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { @@ -30,8 +29,17 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; + let mut contracts = chain_config.get_contracts_config()?; let spinner = Spinner::new(MSG_INITIALIZING_BRIDGES_SPINNER); - initialize_bridges(shell, &chain_config, &ecosystem_config, args).await?; + initialize_bridges( + shell, + &chain_config, + &ecosystem_config, + &mut contracts, + args, + ) + .await?; + contracts.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); Ok(()) @@ -41,6 +49,7 @@ pub async fn initialize_bridges( shell: &Shell, chain_config: &ChainConfig, ecosystem_config: &EcosystemConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { build_l2_contracts(shell, &ecosystem_config.link_to_code)?; @@ -74,7 +83,7 @@ pub async fn initialize_bridges( INITIALIZE_BRIDGES_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l2_shared_bridge(shell, chain_config, &output)?; + contracts_config.set_l2_shared_bridge(&output)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index 759b4aaea557..aabb0d714c53 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod args; -mod create; -pub mod deploy_paymaster; -pub mod genesis; -pub(crate) mod init; -mod initialize_bridges; - pub(crate) use args::create::ChainCreateArgsFinal; use clap::Subcommand; use common::forge::ForgeScriptArgs; @@ -13,6 +6,13 @@ use xshell::Shell; use crate::commands::chain::args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}; +pub(crate) mod args; +mod create; +pub mod deploy_paymaster; +pub mod genesis; +pub(crate) mod init; +mod initialize_bridges; + #[derive(Subcommand, Debug)] pub enum ChainCommands { /// Create a new chain, setting the necessary configurations for later initialization diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs index fecda40c7760..3099b3cf8c27 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs @@ -41,7 +41,6 @@ use crate::{ }, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ msg_ecosystem_initialized, msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, @@ -49,6 +48,7 @@ use crate::{ MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs new file mode 100644 index 000000000000..ebc7855c2b58 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs @@ -0,0 +1,2 @@ +pub mod prepare_configs; +pub mod run; diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs new file mode 100644 index 000000000000..e82fbd7ca155 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs @@ -0,0 +1,69 @@ +use clap::Parser; +use common::{db::DatabaseConfig, Prompt}; +use config::ChainConfig; +use serde::{Deserialize, Serialize}; +use slugify_rs::slugify; +use url::Url; + +use crate::{ + defaults::{generate_external_node_db_name, DATABASE_SERVER_URL, LOCAL_RPC_URL}, + messages::{ + msg_external_node_db_name_prompt, msg_external_node_db_url_prompt, MSG_L1_RPC_URL_PROMPT, + MSG_USE_DEFAULT_DATABASES_HELP, + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Parser, Default)] +pub struct PrepareConfigArgs { + #[clap(long)] + pub db_url: Option, + #[clap(long)] + pub db_name: Option, + #[clap(long)] + pub l1_rpc_url: Option, + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] + pub use_default: bool, +} + +impl PrepareConfigArgs { + pub fn fill_values_with_prompt(self, config: &ChainConfig) -> PrepareConfigFinal { + let db_name = generate_external_node_db_name(config); + let chain_name = config.name.clone(); + if self.use_default { + PrepareConfigFinal { + db: DatabaseConfig::new(DATABASE_SERVER_URL.clone(), db_name), + l1_rpc_url: LOCAL_RPC_URL.to_string(), + } + } else { + let db_url = self.db_url.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_url_prompt(&chain_name)) + .default(DATABASE_SERVER_URL.as_str()) + .ask() + }); + let db_name = slugify!( + &self.db_name.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_name_prompt(&chain_name)) + .default(&db_name) + .ask() + }), + separator = "_" + ); + let l1_rpc_url = self.l1_rpc_url.unwrap_or_else(|| { + Prompt::new(&MSG_L1_RPC_URL_PROMPT) + .default(&LOCAL_RPC_URL) + .ask() + }); + + PrepareConfigFinal { + db: DatabaseConfig::new(db_url, db_name), + l1_rpc_url, + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrepareConfigFinal { + pub db: DatabaseConfig, + pub l1_rpc_url: String, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs new file mode 100644 index 000000000000..1bc0c06728d7 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs @@ -0,0 +1,15 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use crate::messages::{MSG_SERVER_ADDITIONAL_ARGS_HELP, MSG_SERVER_COMPONENTS_HELP}; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct RunExternalNodeArgs { + #[clap(long)] + pub reinit: bool, + #[clap(long, help = MSG_SERVER_COMPONENTS_HELP)] + pub components: Option>, + #[clap(long, short)] + #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] + pub additional_args: Vec, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs new file mode 100644 index 000000000000..c6101e88739c --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs @@ -0,0 +1,53 @@ +use anyhow::Context; +use common::{ + config::global_config, + db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, + spinner::Spinner, +}; +use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; +use xshell::Shell; + +use crate::{ + consts::SERVER_MIGRATIONS, + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED, + MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_INITIALIZING_DATABASES_SPINNER, + }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, +}; + +pub async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + init(shell, &chain_config).await +} + +pub async fn init(shell: &Shell, chain_config: &ChainConfig) -> anyhow::Result<()> { + let spin = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); + let secrets = SecretsConfig::read_with_base_path( + shell, + chain_config + .external_node_config_path + .clone() + .context(MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED)?, + )?; + let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; + drop_db_if_exists(&db_config) + .await + .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; + init_db(&db_config).await?; + recreate_rocksdb_dirs( + shell, + &chain_config.rocks_db_path, + RocksDBDirOption::ExternalNode, + )?; + let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); + migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; + spin.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs new file mode 100644 index 000000000000..06e422de08b8 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs @@ -0,0 +1,24 @@ +use args::{prepare_configs::PrepareConfigArgs, run::RunExternalNodeArgs}; +use clap::Parser; +use serde::{Deserialize, Serialize}; +use xshell::Shell; + +mod args; +mod init; +mod prepare_configs; +mod run; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub enum ExternalNodeCommands { + Configs(PrepareConfigArgs), + Init, + Run(RunExternalNodeArgs), +} + +pub async fn run(shell: &Shell, commands: ExternalNodeCommands) -> anyhow::Result<()> { + match commands { + ExternalNodeCommands::Configs(args) => prepare_configs::run(shell, args), + ExternalNodeCommands::Init => init::run(shell).await, + ExternalNodeCommands::Run(args) => run::run(shell, args).await, + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs new file mode 100644 index 000000000000..4df420474ecb --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -0,0 +1,79 @@ +use std::path::Path; + +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ + external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, DatabaseSecrets, + EcosystemConfig, L1Secret, SecretsConfig, +}; +use xshell::Shell; + +use crate::{ + commands::external_node::args::prepare_configs::{PrepareConfigArgs, PrepareConfigFinal}, + messages::{ + msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, + }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, +}; + +pub fn run(shell: &Shell, args: PrepareConfigArgs) -> anyhow::Result<()> { + logger::info(MSG_PREPARING_EN_CONFIGS); + let chain_name = global_config().chain_name.clone(); + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let mut chain_config = ecosystem_config + .load_chain(chain_name) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + let args = args.fill_values_with_prompt(&chain_config); + let external_node_config_path = chain_config + .external_node_config_path + .unwrap_or_else(|| chain_config.configs.join("external_node")); + shell.create_dir(&external_node_config_path)?; + chain_config.external_node_config_path = Some(external_node_config_path.clone()); + prepare_configs(shell, &chain_config, &external_node_config_path, args)?; + let chain_path = ecosystem_config.chains.join(&chain_config.name); + chain_config.save_with_base_path(shell, chain_path)?; + logger::info(msg_preparing_en_config_is_done(&external_node_config_path)); + Ok(()) +} + +fn prepare_configs( + shell: &Shell, + config: &ChainConfig, + en_configs_path: &Path, + args: PrepareConfigFinal, +) -> anyhow::Result<()> { + let genesis = config.get_genesis_config()?; + let general = config.get_general_config()?; + let en_config = ENConfig { + l2_chain_id: genesis.l2_chain_id, + l1_chain_id: genesis.l1_chain_id, + l1_batch_commit_data_generator_mode: genesis + .l1_batch_commit_data_generator_mode + .unwrap_or_default(), + main_node_url: general.api.web3_json_rpc.http_url.clone(), + main_node_rate_limit_rps: None, + }; + let mut general_en = general.clone(); + general_en.update_ports(&general.ports_config().next_empty_ports_config())?; + let secrets = SecretsConfig { + database: DatabaseSecrets { + server_url: args.db.full_url(), + prover_url: None, + other: Default::default(), + }, + l1: L1Secret { + l1_rpc_url: args.l1_rpc_url.clone(), + other: Default::default(), + }, + other: Default::default(), + }; + secrets.save_with_base_path(shell, en_configs_path)?; + let dirs = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::ExternalNode)?; + general_en.set_rocks_db_config(dirs)?; + + general_en.save_with_base_path(shell, &en_configs_path)?; + en_config.save_with_base_path(shell, &en_configs_path)?; + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs new file mode 100644 index 000000000000..9d3da4663859 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs @@ -0,0 +1,37 @@ +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::Shell; + +use crate::{ + commands::external_node::{args::run::RunExternalNodeArgs, init}, + external_node::RunExternalNode, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_EN}, +}; + +pub async fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + logger::info(MSG_STARTING_EN); + + run_external_node(args, &chain_config, shell).await?; + + Ok(()) +} + +async fn run_external_node( + args: RunExternalNodeArgs, + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result<()> { + if args.reinit { + init::init(shell, chain_config).await? + } + let server = RunExternalNode::new(args.components.clone(), chain_config)?; + server.run(shell, args.additional_args.clone()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index ccdf5b082caa..db34e1d8647d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -2,5 +2,6 @@ pub mod args; pub mod chain; pub mod containers; pub mod ecosystem; +pub mod external_node; pub mod prover; pub mod server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/server.rs b/zk_toolbox/crates/zk_inception/src/commands/server.rs index e2d35dd9b792..aed16357c925 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/server.rs @@ -1,11 +1,11 @@ use anyhow::Context; -use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; +use common::{config::global_config, logger}; use config::{ChainConfig, EcosystemConfig}; -use xshell::{cmd, Shell}; +use xshell::Shell; use crate::{ commands::args::RunServerArgs, - messages::{MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, server::{RunServer, ServerMode}, }; @@ -19,20 +19,11 @@ pub fn run(shell: &Shell, args: RunServerArgs) -> anyhow::Result<()> { logger::info(MSG_STARTING_SERVER); - build_l1_contracts(shell, &ecosystem_config)?; run_server(args, &chain_config, shell)?; Ok(()) } -fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { - let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); - let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); - Cmd::new(cmd!(shell, "yarn build")).run()?; - spinner.finish(); - Ok(()) -} - fn run_server( args: RunServerArgs, chain_config: &ChainConfig, @@ -44,5 +35,5 @@ fn run_server( } else { ServerMode::Normal }; - server.run(shell, mode) + server.run(shell, mode, args.additional_args) } diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index a300a15e76c6..e69de29bb2d1 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -1,97 +0,0 @@ -use common::db::DatabaseConfig; -use config::{ - forge_interface::{ - initialize_bridges::output::InitializeBridgeOutput, paymaster::DeployPaymasterOutput, - register_chain::output::RegisterChainOutput, - }, - traits::{ReadConfigWithBasePath, SaveConfigWithBasePath}, - ChainConfig, ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, -}; -use types::ProverMode; -use xshell::Shell; - -use crate::defaults::{ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE}; - -pub(crate) fn update_genesis(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut genesis = GenesisConfig::read_with_base_path(shell, &config.configs)?; - - genesis.l2_chain_id = config.chain_id; - genesis.l1_chain_id = config.l1_network.chain_id(); - genesis.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); - - genesis.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_database_secrets( - shell: &Shell, - config: &ChainConfig, - server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.database.server_url = server_db_config.full_url(); - secrets.database.prover_url = prover_db_config.full_url(); - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_l1_rpc_url_secret( - shell: &Shell, - config: &ChainConfig, - l1_rpc_url: String, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.l1.l1_rpc_url = l1_rpc_url; - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_general_config(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut general = GeneralConfig::read_with_base_path(shell, &config.configs)?; - general.db.state_keeper_db_path = - shell.create_dir(config.rocks_db_path.join(ROCKS_DB_STATE_KEEPER))?; - general.db.merkle_tree.path = shell.create_dir(config.rocks_db_path.join(ROCKS_DB_TREE))?; - if config.prover_version != ProverMode::NoProofs { - general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); - } - general.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_l1_contracts( - shell: &Shell, - config: &ChainConfig, - register_chain_output: &RegisterChainOutput, -) -> anyhow::Result { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; - contracts_config.l1.governance_addr = register_chain_output.governance_addr; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(contracts_config) -} - -pub fn update_l2_shared_bridge( - shell: &Shell, - config: &ChainConfig, - initialize_bridges_output: &InitializeBridgeOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.bridges.shared.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.bridges.erc20.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_paymaster( - shell: &Shell, - config: &ChainConfig, - paymaster_output: &DeployPaymasterOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l2.testnet_paymaster_addr = paymaster_output.paymaster; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index a59024d09b40..8dde9337a73f 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -1,3 +1,5 @@ pub const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; pub const MINIMUM_BALANCE_FOR_WALLET: u128 = 5000000000000000000; +pub const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; +pub const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 04b735e02275..40be1293614b 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -9,8 +9,10 @@ lazy_static! { Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); } -pub const ROCKS_DB_STATE_KEEPER: &str = "main/state_keeper"; -pub const ROCKS_DB_TREE: &str = "main/tree"; +pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; +pub const ROCKS_DB_TREE: &str = "tree"; +pub const EN_ROCKS_DB_PREFIX: &str = "en"; +pub const MAIN_ROCKS_DB_PREFIX: &str = "main"; pub const L2_CHAIN_ID: u32 = 271; /// Path to base chain configuration inside zksync-era @@ -36,3 +38,11 @@ pub fn generate_db_names(config: &ChainConfig) -> DBNames { ), } } + +pub fn generate_external_node_db_name(config: &ChainConfig) -> String { + format!( + "external_node_{}_{}", + config.l1_network.to_string().to_ascii_lowercase(), + config.name + ) +} diff --git a/zk_toolbox/crates/zk_inception/src/external_node.rs b/zk_toolbox/crates/zk_inception/src/external_node.rs new file mode 100644 index 000000000000..baf00cccae5f --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/external_node.rs @@ -0,0 +1,77 @@ +use std::path::PathBuf; + +use anyhow::Context; +use common::cmd::Cmd; +use config::{ + external_node::ENConfig, traits::FileConfigWithDefaultName, ChainConfig, GeneralConfig, + SecretsConfig, +}; +use xshell::{cmd, Shell}; + +use crate::messages::MSG_FAILED_TO_RUN_SERVER_ERR; + +pub struct RunExternalNode { + components: Option>, + code_path: PathBuf, + general_config: PathBuf, + secrets: PathBuf, + en_config: PathBuf, +} + +impl RunExternalNode { + pub fn new( + components: Option>, + chain_config: &ChainConfig, + ) -> anyhow::Result { + let en_path = chain_config + .external_node_config_path + .clone() + .context("External node is not initialized")?; + let general_config = GeneralConfig::get_path_with_base_path(&en_path); + let secrets = SecretsConfig::get_path_with_base_path(&en_path); + let enconfig = ENConfig::get_path_with_base_path(&en_path); + + Ok(Self { + components, + code_path: chain_config.link_to_code.clone(), + general_config, + secrets, + en_config: enconfig, + }) + } + + pub fn run(&self, shell: &Shell, mut additional_args: Vec) -> anyhow::Result<()> { + shell.change_dir(&self.code_path); + let config_general_config = &self.general_config.to_str().unwrap(); + let en_config = &self.en_config.to_str().unwrap(); + let secrets = &self.secrets.to_str().unwrap(); + if let Some(components) = self.components() { + additional_args.push(format!("--components={}", components)) + } + let mut cmd = Cmd::new( + cmd!( + shell, + "cargo run --release --bin zksync_external_node -- + --config-path {config_general_config} + --secrets-path {secrets} + --external-node-config-path {en_config} + " + ) + .args(additional_args) + .env_remove("RUSTUP_TOOLCHAIN"), + ) + .with_force_run(); + + cmd.run().context(MSG_FAILED_TO_RUN_SERVER_ERR)?; + Ok(()) + } + + fn components(&self) -> Option { + self.components.as_ref().and_then(|components| { + if components.is_empty() { + return None; + } + Some(components.join(",")) + }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index dff9e479e01f..f381ad7fb47c 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -8,17 +8,18 @@ use config::EcosystemConfig; use xshell::Shell; use crate::commands::{ - args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, prover::ProverCommands, + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, + external_node::ExternalNodeCommands, prover::ProverCommands, }; pub mod accept_ownership; mod commands; -mod config_manipulations; mod consts; mod defaults; -pub mod forge_utils; +pub mod external_node; mod messages; pub mod server; +mod utils; #[derive(Parser, Debug)] #[command(version, about)] @@ -42,6 +43,9 @@ pub enum InceptionSubcommands { Prover(ProverCommands), /// Run server Server(RunServerArgs), + // Run External Node + #[command(subcommand)] + ExternalNode(ExternalNodeCommands), /// Run containers for local development Containers, } @@ -109,6 +113,9 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, + InceptionSubcommands::ExternalNode(args) => { + commands::external_node::run(shell, args).await? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 1b3c05258753..1fa36fbabb1b 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use ethers::{ types::{H160, U256}, utils::format_ether, @@ -43,7 +45,6 @@ pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT: &str = "Provide the path t pub(super) const MSG_L1_RPC_URL_INVALID_ERR: &str = "Invalid RPC URL"; pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR: &str = "Invalid path"; pub(super) const MSG_GENESIS_DATABASE_ERR: &str = "Unable to perform genesis on the database"; -pub(super) const MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR: &str = "Ecosystem contracts config not found"; pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; pub(super) const MSG_INITIALIZING_ECOSYSTEM: &str = "Initializing ecosystem"; pub(super) const MSG_DEPLOYING_ERC20: &str = "Deploying ERC20 contracts"; @@ -55,6 +56,7 @@ pub(super) const MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER: &str = "Deploying ecosystem contracts..."; pub(super) const MSG_REGISTERING_CHAIN_SPINNER: &str = "Registering chain..."; pub(super) const MSG_ACCEPTING_ADMIN_SPINNER: &str = "Accepting admin..."; +pub(super) const MSG_RECREATE_ROCKS_DB_ERRROR: &str = "Failed to create rocks db path"; pub(super) fn msg_initializing_chain(chain_name: &str) -> String { format!("Initializing chain {chain_name}") @@ -118,7 +120,7 @@ pub(super) const MSG_SERVER_DB_URL_HELP: &str = "Server database url without dat pub(super) const MSG_SERVER_DB_NAME_HELP: &str = "Server database name"; pub(super) const MSG_PROVER_DB_URL_HELP: &str = "Prover database url without database name"; pub(super) const MSG_PROVER_DB_NAME_HELP: &str = "Prover database name"; -pub(super) const MSG_GENESIS_USE_DEFAULT_HELP: &str = "Use default database urls and names"; +pub(super) const MSG_USE_DEFAULT_DATABASES_HELP: &str = "Use default database urls and names"; pub(super) const MSG_GENESIS_COMPLETED: &str = "Genesis completed successfully"; pub(super) const MSG_STARTING_GENESIS: &str = "Starting genesis process"; pub(super) const MSG_INITIALIZING_DATABASES_SPINNER: &str = "Initializing databases..."; @@ -133,6 +135,10 @@ pub(super) fn msg_server_db_url_prompt(chain_name: &str) -> String { format!("Please provide server database url for chain {chain_name}") } +pub(super) fn msg_external_node_db_url_prompt(chain_name: &str) -> String { + format!("Please provide external_node database url for chain {chain_name}") +} + pub(super) fn msg_prover_db_url_prompt(chain_name: &str) -> String { format!("Please provide prover database url for chain {chain_name}") } @@ -141,6 +147,10 @@ pub(super) fn msg_prover_db_name_prompt(chain_name: &str) -> String { format!("Please provide prover database name for chain {chain_name}") } +pub(super) fn msg_external_node_db_name_prompt(chain_name: &str) -> String { + format!("Please provide external_node database name for chain {chain_name}") +} + pub(super) fn msg_server_db_name_prompt(chain_name: &str) -> String { format!("Please provide server database name for chain {chain_name}") } @@ -173,6 +183,7 @@ pub(super) const MSG_FAILED_TO_FIND_ECOSYSTEM_ERR: &str = "Failed to find ecosys pub(super) const MSG_STARTING_SERVER: &str = "Starting server"; pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts..."; +pub(super) const MSG_PREPARING_EN_CONFIGS: &str = "Preparing External Node config"; /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; @@ -189,6 +200,14 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( ) } +pub(super) fn msg_preparing_en_config_is_done(path: &Path) -> String { + format!("External nodes configs could be found in: {path:?}") +} + +pub(super) const MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED: &str = + "External node is not initialized"; /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; + +pub(super) const MSG_STARTING_EN: &str = "Starting external node"; diff --git a/zk_toolbox/crates/zk_inception/src/server.rs b/zk_toolbox/crates/zk_inception/src/server.rs index 6773d224cba3..c4feb1c7c272 100644 --- a/zk_toolbox/crates/zk_inception/src/server.rs +++ b/zk_toolbox/crates/zk_inception/src/server.rs @@ -44,14 +44,18 @@ impl RunServer { } } - pub fn run(&self, shell: &Shell, server_mode: ServerMode) -> anyhow::Result<()> { + pub fn run( + &self, + shell: &Shell, + server_mode: ServerMode, + mut additional_args: Vec, + ) -> anyhow::Result<()> { shell.change_dir(&self.code_path); let config_genesis = &self.genesis.to_str().unwrap(); let config_wallets = &self.wallets.to_str().unwrap(); let config_general_config = &self.general_config.to_str().unwrap(); let config_contracts = &self.contracts.to_str().unwrap(); let secrets = &self.secrets.to_str().unwrap(); - let mut additional_args = vec![]; if let Some(components) = self.components() { additional_args.push(format!("--components={}", components)) } diff --git a/zk_toolbox/crates/zk_inception/src/forge_utils.rs b/zk_toolbox/crates/zk_inception/src/utils/forge.rs similarity index 100% rename from zk_toolbox/crates/zk_inception/src/forge_utils.rs rename to zk_toolbox/crates/zk_inception/src/utils/forge.rs diff --git a/zk_toolbox/crates/zk_inception/src/utils/mod.rs b/zk_toolbox/crates/zk_inception/src/utils/mod.rs new file mode 100644 index 000000000000..a84f0a336de5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod forge; +pub mod rocks_db; diff --git a/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs new file mode 100644 index 000000000000..fc80aca100bc --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs @@ -0,0 +1,39 @@ +use std::path::Path; + +use config::RocksDbs; +use xshell::Shell; + +use crate::defaults::{ + EN_ROCKS_DB_PREFIX, MAIN_ROCKS_DB_PREFIX, ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE, +}; + +pub enum RocksDBDirOption { + Main, + ExternalNode, +} + +impl RocksDBDirOption { + pub fn prefix(&self) -> &str { + match self { + RocksDBDirOption::Main => MAIN_ROCKS_DB_PREFIX, + RocksDBDirOption::ExternalNode => EN_ROCKS_DB_PREFIX, + } + } +} + +pub fn recreate_rocksdb_dirs( + shell: &Shell, + rocks_db_path: &Path, + option: RocksDBDirOption, +) -> anyhow::Result { + let state_keeper = rocks_db_path + .join(option.prefix()) + .join(ROCKS_DB_STATE_KEEPER); + shell.remove_path(&state_keeper)?; + let merkle_tree = rocks_db_path.join(option.prefix()).join(ROCKS_DB_TREE); + shell.remove_path(&merkle_tree)?; + Ok(RocksDbs { + state_keeper: shell.create_dir(state_keeper)?, + merkle_tree: shell.create_dir(merkle_tree)?, + }) +} diff --git a/zk_toolbox/crates/zk_supervisor/Cargo.toml b/zk_toolbox/crates/zk_supervisor/Cargo.toml index 79d2bac74905..d8f5d7862a04 100644 --- a/zk_toolbox/crates/zk_supervisor/Cargo.toml +++ b/zk_toolbox/crates/zk_supervisor/Cargo.toml @@ -21,3 +21,4 @@ strum_macros.workspace = true tokio.workspace = true url.workspace = true xshell.workspace = true +serde.workspace = true diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs index c5b1229dd2ce..c506f7d07894 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs @@ -1,30 +1,54 @@ -use common::{cmd::Cmd, logger, spinner::Spinner}; +use clap::Parser; +use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; use config::EcosystemConfig; +use serde::{Deserialize, Serialize}; use xshell::{cmd, Shell}; use crate::messages::{ - MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES, - MSG_INTEGRATION_TESTS_RUN_INFO, MSG_INTEGRATION_TESTS_RUN_SUCCESS, + msg_integration_tests_run, MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, + MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES, MSG_INTEGRATION_TESTS_RUN_SUCCESS, }; +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct IntegrationTestCommands { + #[clap(short, long)] + external_node: bool, +} + const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; const CONTRACTS_TEST_DATA_PATH: &str = "etc/contracts-test-data"; -pub fn run(shell: &Shell) -> anyhow::Result<()> { +pub fn run( + shell: &Shell, + integration_test_commands: IntegrationTestCommands, +) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); - logger::info(MSG_INTEGRATION_TESTS_RUN_INFO); + logger::info(msg_integration_tests_run( + integration_test_commands.external_node, + )); build_repository(shell, &ecosystem_config)?; build_test_contracts(shell, &ecosystem_config)?; - Cmd::new( - cmd!(shell, "yarn jest --forceExit --testTimeout 60000") - .env("CHAIN_NAME", ecosystem_config.default_chain), - ) - .with_force_run() - .run()?; + let mut command = cmd!(shell, "yarn jest --forceExit --testTimeout 60000") + .env("CHAIN_NAME", ecosystem_config.default_chain); + + if integration_test_commands.external_node { + command = command.env( + "EXTERNAL_NODE", + format!("{:?}", integration_test_commands.external_node), + ) + } + if global_config().verbose { + command = command.env( + "ZKSYNC_DEBUG_LOGS", + format!("{:?}", global_config().verbose), + ) + } + + Cmd::new(command).with_force_run().run()?; logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS); diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index f2f6f86cfc61..ae8815c96899 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -1,10 +1,10 @@ -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use common::config::global_config; use config::{EcosystemConfig, SecretsConfig}; use url::Url; use xshell::Shell; -use crate::messages::MSG_CHAIN_NOT_FOUND_ERR; +use crate::messages::{MSG_CHAIN_NOT_FOUND_ERR, MSG_PROVER_URL_MUST_BE_PRESENTED}; const CORE_DAL_PATH: &str = "core/lib/dal"; const PROVER_DAL_PATH: &str = "prover/prover_dal"; @@ -46,7 +46,11 @@ pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { Ok(Dal { path: PROVER_DAL_PATH.to_string(), - url: secrets.database.prover_url.clone(), + url: secrets + .database + .prover_url + .context(MSG_PROVER_URL_MUST_BE_PRESENTED)? + .clone(), }) } diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index ab5629465a88..96ab59bdad10 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -12,6 +12,8 @@ use messages::{ }; use xshell::Shell; +use crate::commands::integration_tests::IntegrationTestCommands; + mod commands; mod dals; mod messages; @@ -30,7 +32,7 @@ enum SupervisorSubcommands { #[command(subcommand, about = MSG_SUBCOMMAND_DATABASE_ABOUT)] Database(DatabaseCommands), #[command(about = MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT)] - IntegrationTests, + IntegrationTests(IntegrationTestCommands), } #[derive(Parser, Debug)] @@ -93,7 +95,9 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { match args.command { SupervisorSubcommands::Database(command) => commands::database::run(shell, command).await?, - SupervisorSubcommands::IntegrationTests => commands::integration_tests::run(shell)?, + SupervisorSubcommands::IntegrationTests(args) => { + commands::integration_tests::run(shell, args)? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 31bdb0eb9b1d..7ef956b8f545 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -1,5 +1,6 @@ // Ecosystem related messages pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; + pub(super) fn msg_global_chain_does_not_exist(chain: &str, available_chains: &str) -> String { format!("Chain with name {chain} doesnt exist, please choose one of: {available_chains}") } @@ -10,12 +11,15 @@ pub(super) const MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT: &str = "Run integration // Database related messages pub(super) const MSG_NO_DATABASES_SELECTED: &str = "No databases selected"; + pub(super) fn msg_database_info(gerund_verb: &str) -> String { format!("{gerund_verb} databases") } + pub(super) fn msg_database_success(past_verb: &str) -> String { format!("Databases {past_verb} successfully") } + pub(super) fn msg_database_loading(gerund_verb: &str, dal: &str) -> String { format!("{gerund_verb} database for dal {dal}...") } @@ -33,6 +37,8 @@ pub(super) const MSG_DATABASE_RESET_PAST: &str = "reset"; pub(super) const MSG_DATABASE_SETUP_GERUND: &str = "Setting up"; pub(super) const MSG_DATABASE_SETUP_PAST: &str = "set up"; +pub(super) const MSG_PROVER_URL_MUST_BE_PRESENTED: &str = "Prover url must be presented"; + pub(super) const MSG_DATABASE_COMMON_PROVER_HELP: &str = "Prover database"; pub(super) const MSG_DATABASE_COMMON_CORE_HELP: &str = "Core database"; pub(super) const MSG_DATABASE_NEW_MIGRATION_DATABASE_HELP: &str = @@ -57,13 +63,24 @@ pub(super) const MSG_DATABASE_NEW_MIGRATION_DB_PROMPT: &str = "What database do you want to create a new migration for?"; pub(super) const MSG_DATABASE_NEW_MIGRATION_NAME_PROMPT: &str = "How do you want to name the migration?"; + pub(super) fn msg_database_new_migration_loading(dal: &str) -> String { format!("Creating new database migration for dal {}...", dal) } + pub(super) const MSG_DATABASE_NEW_MIGRATION_SUCCESS: &str = "Migration created successfully"; // Integration tests related messages -pub(super) const MSG_INTEGRATION_TESTS_RUN_INFO: &str = "Running integration tests"; + +pub(super) fn msg_integration_tests_run(external_node: bool) -> String { + let base = "Running integration tests"; + if external_node { + format!("{} for external node", base) + } else { + format!("{} for main server", base) + } +} + pub(super) const MSG_INTEGRATION_TESTS_RUN_SUCCESS: &str = "Integration tests ran successfully"; pub(super) const MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES: &str = "Building repository dependencies...";