From 4fdac8208b990118cbed8c1b7000583e44bb4f83 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 30 May 2024 12:12:01 +0200 Subject: [PATCH 01/56] File based config for en Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 + core/lib/config/src/configs/en_config.rs | 10 +++++++ core/lib/config/src/configs/mod.rs | 1 + core/lib/protobuf_config/src/en.rs | 29 +++++++++++++++++++ core/lib/protobuf_config/src/lib.rs | 1 + .../protobuf_config/src/proto/config/en.proto | 10 +++++++ core/lib/protobuf_config/src/tests.rs | 1 + etc/env/file_based/external_node.yaml | 3 ++ 8 files changed, 56 insertions(+) create mode 100644 core/lib/config/src/configs/en_config.rs create mode 100644 core/lib/protobuf_config/src/en.rs create mode 100644 core/lib/protobuf_config/src/proto/config/en.proto create mode 100644 etc/env/file_based/external_node.yaml diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 56d66a3a4253..b00e78a93b3c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -2,6 +2,7 @@ use std::{ env, ffi::OsString, num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + path::PathBuf, time::Duration, }; diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs new file mode 100644 index 000000000000..287a47ce4eaa --- /dev/null +++ b/core/lib/config/src/configs/en_config.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; +use zksync_basic_types::{L1ChainId, L2ChainId}; + +/// Temporary config for initializing external node, will be completely replaced by consensus config later +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ENConfig { + pub l2_chain_id: L2ChainId, + pub l1_chain_id: L1ChainId, + pub main_node_url: String, +} diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 925c30976f97..148a7b2c61a0 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -28,6 +28,7 @@ pub mod consensus; pub mod contract_verifier; pub mod contracts; pub mod database; +pub mod en_config; pub mod eth_sender; pub mod eth_watch; mod experimental; diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs new file mode 100644 index 000000000000..afc1a50e3723 --- /dev/null +++ b/core/lib/protobuf_config/src/en.rs @@ -0,0 +1,29 @@ +use zksync_basic_types::{L1ChainId, L2ChainId}; +use zksync_config::configs::en_config::ENConfig; +use zksync_protobuf::{required, ProtoRepr}; + +use crate::proto::en as proto; + +impl ProtoRepr for proto::ExternalNode { + type Type = ENConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + main_node_url: required(&self.main_node_url)?.to_string(), + l1_chain_id: required(&self.l1_chain_id) + .map(|x| L1ChainId(*x)) + .context("l1_chain_id")?, + l2_chain_id: required(&self.l2_chain_id) + .and_then(|x| L2ChainId::try_from(*x).map_err(|a| anyhow::anyhow!(a))) + .context("l2_chain_id")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + main_node_url: Some(this.main_node_url.clone()), + l1_chain_id: Some(this.l1_chain_id.0), + l2_chain_id: Some(this.l2_chain_id.as_u64()), + } + } +} diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index 25d5662b9ddb..6eca9be9c2e3 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -11,6 +11,7 @@ mod consensus; mod contract_verifier; mod contracts; mod database; +mod en; mod eth; mod experimental; mod general; diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto new file mode 100644 index 000000000000..ed2553f5aca5 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.en; + + +message ExternalNode { + optional string main_node_url = 1; // required + optional uint64 l2_chain_id = 2; // required + optional uint64 l1_chain_id = 3; // required +} diff --git a/core/lib/protobuf_config/src/tests.rs b/core/lib/protobuf_config/src/tests.rs index fad37700ae5f..f5a36b2f4e30 100644 --- a/core/lib/protobuf_config/src/tests.rs +++ b/core/lib/protobuf_config/src/tests.rs @@ -65,4 +65,5 @@ fn verify_file_parsing() { decode_yaml_repr::(&base_path.join("contracts.yaml"), true) .unwrap(); decode_yaml_repr::(&base_path.join("secrets.yaml"), true).unwrap(); + decode_yaml_repr::(&base_path.join("secrets.yaml"), true).unwrap(); } diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml new file mode 100644 index 000000000000..a909fdfa9ba0 --- /dev/null +++ b/etc/env/file_based/external_node.yaml @@ -0,0 +1,3 @@ +l1_chain_id: 9 +l2_chain_id: 270 +main_node_url: http://localhost:3050 \ No newline at end of file From d232f27f831320074de4b36981c819bd88c717b7 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 30 May 2024 19:24:28 +0200 Subject: [PATCH 02/56] Implement external node config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 259 +++++++++++++++++- .../external_node/src/config/observability.rs | 29 ++ core/lib/config/src/configs/api.rs | 4 + core/lib/config/src/configs/en_config.rs | 28 +- core/lib/config/src/configs/experimental.rs | 6 + core/lib/config/src/testonly.rs | 5 + core/lib/protobuf_config/src/api.rs | 10 +- core/lib/protobuf_config/src/en.rs | 88 +++++- core/lib/protobuf_config/src/experimental.rs | 10 + core/lib/protobuf_config/src/genesis.rs | 5 +- .../src/proto/config/api.proto | 3 +- .../protobuf_config/src/proto/config/en.proto | 18 ++ .../src/proto/config/experimental.proto | 3 + .../src/temp_config_store/mod.rs | 9 + etc/env/file_based/external_node.yaml | 3 +- etc/env/file_based/general.yaml | 2 +- 16 files changed, 468 insertions(+), 14 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 56d66a3a4253..150c9c4cee5f 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -2,19 +2,23 @@ use std::{ env, ffi::OsString, num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + path::PathBuf, time::Duration, }; use anyhow::Context; +use futures::StreamExt; use serde::Deserialize; use zksync_config::{ configs::{ api::{MaxResponseSize, MaxResponseSizeOverrides}, consensus::{ConsensusConfig, ConsensusSecrets}, + en_config::ENConfig, + GeneralConfig, Secrets, }, ObjectStoreConfig, }; -use zksync_core_leftovers::temp_config_store::decode_yaml_repr; +use zksync_core_leftovers::temp_config_store::{decode_yaml_repr, read_yaml_repr}; #[cfg(test)] use zksync_dal::{ConnectionPool, Core}; use zksync_metadata_calculator::MetadataCalculatorRecoveryConfig; @@ -415,6 +419,134 @@ pub(crate) struct OptionalENConfig { } impl OptionalENConfig { + fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { + let api = general_config + .api_config + .as_ref() + .context("Api is required")?; + let db = general_config + .db_config + .as_ref() + .context("Db config is required")?; + let postgres = general_config + .postgres_config + .as_ref() + .context("Db config is required")?; + let state_keeper = general_config + .state_keeper_config + .as_ref() + .context("Db config is required")?; + + let api_namespaces = api + .web3_json_rpc + .api_namespaces + .as_ref() + .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + + Ok(OptionalENConfig { + filters_limit: api + .web3_json_rpc + .filters_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_filters_limit), + subscriptions_limit: api + .web3_json_rpc + .subscriptions_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_subscriptions_limit), + req_entities_limit: api + .web3_json_rpc + .req_entities_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_req_entities_limit), + max_tx_size_bytes: api.web3_json_rpc.max_tx_size, + vm_execution_cache_misses_limit: api.web3_json_rpc.vm_execution_cache_misses_limit, + transactions_per_sec_limit: None, + fee_history_limit: api.web3_json_rpc.fee_history_limit(), + max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), + max_response_body_size_mb: api.web3_json_rpc.max_response_body_size().global, + max_response_body_size_overrides_mb: api + .web3_json_rpc + .max_response_body_size() + .overrides, + pubsub_polling_interval_ms: api.web3_json_rpc.pubsub_interval().as_millis() as u64, + max_nonce_ahead: api.web3_json_rpc.max_nonce_ahead, + vm_concurrency_limit: api.web3_json_rpc.vm_concurrency_limit(), + factory_deps_cache_size_mb: api.web3_json_rpc.factory_deps_cache_size(), + initial_writes_cache_size_mb: api.web3_json_rpc.initial_writes_cache_size(), + latest_values_cache_size_mb: api.web3_json_rpc.latest_values_cache_size(), + filters_disabled: api.web3_json_rpc.filters_disabled, + mempool_cache_update_interval_ms: api + .web3_json_rpc + .mempool_cache_update_interval() + .as_millis() as u64, + mempool_cache_size: api.web3_json_rpc.mempool_cache_size(), + + healthcheck_slow_time_limit_ms: api.healthcheck.slow_time_limit_ms, + healthcheck_hard_time_limit_ms: api.healthcheck.hard_time_limit_ms, + estimate_gas_scale_factor: api.web3_json_rpc.gas_price_scale_factor, + estimate_gas_acceptable_overestimation: api + .web3_json_rpc + .estimate_gas_acceptable_overestimation, + gas_price_scale_factor: api.web3_json_rpc.gas_price_scale_factor, + merkle_tree_max_l1_batches_per_iter: db.merkle_tree.max_l1_batches_per_iter, + merkle_tree_max_open_files: db.experimental.state_keeper_db_max_open_files, + merkle_tree_multi_get_chunk_size: db.merkle_tree.multi_get_chunk_size, + merkle_tree_block_cache_size_mb: db + .experimental + .state_keeper_db_block_cache_capacity_mb, + merkle_tree_memtable_capacity_mb: db.merkle_tree.memtable_capacity_mb, + merkle_tree_stalled_writes_timeout_sec: db.merkle_tree.stalled_writes_timeout_sec, + database_long_connection_threshold_ms: postgres.long_connection_threshold_ms, + database_slow_query_threshold_ms: postgres.slow_query_threshold_ms, + l2_block_seal_queue_capacity: state_keeper.l2_block_seal_queue_capacity, + main_node_rate_limit_rps: Self::default_main_node_rate_limit_rps(), + l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, + snapshots_recovery_enabled: enconfig + .snapshot_recovery + .as_ref() + .map(|a| a.enabled) + .unwrap_or_default(), + snapshots_recovery_postgres_max_concurrency: enconfig + .snapshot_recovery + .as_ref() + .map(|a| a.postgres_max_concurrency) + .flatten() + .unwrap_or_else(Self::default_snapshots_recovery_postgres_max_concurrency), + pruning_enabled: enconfig + .pruning + .as_ref() + .map(|a| a.enabled) + .unwrap_or_default(), + pruning_chunk_size: enconfig + .pruning + .as_ref() + .map(|a| a.chunk_size) + .flatten() + .unwrap_or_else(Self::default_pruning_chunk_size), + pruning_removal_delay_sec: enconfig + .pruning + .as_ref() + .map(|a| a.removal_delay_sec) + .flatten() + .unwrap_or_else(Self::default_pruning_removal_delay_sec), + pruning_data_retention_sec: enconfig + .pruning + .as_ref() + .map(|a| a.data_retention_sec) + .flatten() + .unwrap_or_else(Self::default_pruning_data_retention_sec), + contracts_diamond_proxy_addr: None, + protective_reads_persistence_enabled: db.experimental.reads_persistence_enabled, + merkle_tree_processing_delay_ms: db.experimental.processing_delay_ms, + merkle_tree_include_indices_and_filters_in_block_cache: db + .experimental + .include_indices_and_filters_in_block_cache, + api_namespaces, + extended_rpc_tracing: api.web3_json_rpc.extended_api_tracing, + }) + } + const fn default_filters_limit() -> usize { 10_000 } @@ -676,6 +808,34 @@ impl RequiredENConfig { .context("could not load external node config") } + fn from_configs( + general: &GeneralConfig, + en_config: &ENConfig, + secrets: &Secrets, + ) -> anyhow::Result { + let api_config = general + .api_config + .as_ref() + .context("Api config is required")?; + let db_config = general.db_config.as_ref().context("Database config")?; + Ok(RequiredENConfig { + l1_chain_id: en_config.l1_chain_id, + l2_chain_id: en_config.l2_chain_id, + http_port: api_config.web3_json_rpc.http_port, + ws_port: api_config.web3_json_rpc.ws_port, + healthcheck_port: api_config.healthcheck.port, + eth_client_url: secrets + .l1 + .as_ref() + .context("Eth config is required")? + .l1_rpc_url + .clone(), + main_node_url: en_config.main_node_url.clone(), + state_cache_path: db_config.state_keeper_db_path.clone(), + merkle_tree_path: db_config.merkle_tree.path.clone(), + }) + } + #[cfg(test)] fn mock(temp_dir: &tempfile::TempDir) -> Self { Self { @@ -786,6 +946,26 @@ impl ExperimentalENConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } + pub fn from_configs( + general_config: &GeneralConfig, + enconfig: &ENConfig, + ) -> anyhow::Result { + let db_config = general_config.db_config.as_ref().context("db_config")?; + let snapshot_recovery = enconfig + .snapshot_recovery + .as_ref() + .context("snapshot_recovery")?; + Ok(Self { + state_keeper_db_block_cache_capacity_mb: db_config + .experimental + .state_keeper_db_block_cache_capacity_mb, + state_keeper_db_max_open_files: db_config.experimental.state_keeper_db_max_open_files, + snapshots_recovery_tree_chunk_size: snapshot_recovery + .tree_chunk_size + .unwrap_or_else(Self::default_snapshots_recovery_tree_chunk_size), + commitment_generator_max_parallelism: enconfig.commitment_generator_max_parallelism, + }) + } } pub(crate) fn read_consensus_secrets() -> anyhow::Result> { @@ -834,11 +1014,29 @@ pub struct ApiComponentConfig { pub tree_api_remote_url: Option, } +impl ApiComponentConfig { + fn from_configs(en_config: &ENConfig) -> Self { + ApiComponentConfig { + tree_api_remote_url: en_config.tree_api_remote_url.clone(), + } + } +} + #[derive(Debug, Deserialize)] pub struct TreeComponentConfig { pub api_port: Option, } +impl TreeComponentConfig { + fn from_configs(general_config: &GeneralConfig) -> Self { + let api_port = general_config + .api_config + .as_ref() + .map(|a| a.merkle_tree.port); + TreeComponentConfig { api_port } + } +} + /// External Node Config contains all the configuration required for the EN operation. /// It is split into three parts: required, optional and remote for easier navigation. #[derive(Debug)] @@ -876,6 +1074,65 @@ impl ExternalNodeConfig<()> { }) } + pub fn from_files( + general_config_path: PathBuf, + external_node_config_patch: PathBuf, + secrets_configs: PathBuf, + consensus: Option, + ) -> anyhow::Result { + let general_config = read_yaml_repr::(general_config_path) + .context("failed decoding general YAML config")?; + let external_node_config = + read_yaml_repr::(external_node_config_patch) + .context("failed decoding external node YAML config")?; + let secrets_config = read_yaml_repr::(secrets_configs) + .context("failed decoding secrets YAML config")?; + + let consensus = consensus + .map(|path| read_yaml_repr::(path)) + .transpose() + .context("failed decoding consensus YAML config")?; + + let required = RequiredENConfig::from_configs( + &general_config, + &external_node_config, + &secrets_config, + )?; + let optional = OptionalENConfig::from_configs(&general_config, &external_node_config)?; + let postgres = PostgresConfig { + database_url: secrets_config + .database + .as_ref() + .context("DB secrets is required")? + .server_url + .clone() + .context("Server url is required")?, + max_connections: general_config + .postgres_config + .as_ref() + .context("Postgres config is required")? + .max_connections()?, + }; + let observability = ObservabilityENConfig::from_configs(&general_config)?; + let experimental = + ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; + + let api_component = ApiComponentConfig::from_configs(&external_node_config); + let tree_component = TreeComponentConfig::from_configs(&general_config); + + Ok(Self { + required, + postgres, + optional, + observability, + experimental, + consensus, + api_component, + tree_component, + remote: (), + }) + } + /// Fetches contracts addresses from the main node, completing the configuration. pub async fn fetch_remote( self, diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index a571b071b5e2..eec8bf2d76a7 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use serde::Deserialize; use vlog::LogFormat; +use zksync_config::configs::{GeneralConfig, ObservabilityConfig}; use super::{ConfigurationSource, Environment}; @@ -98,4 +99,32 @@ impl ObservabilityENConfig { } Ok(guard) } + + pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { + let observability = general_config + .observability + .as_ref() + .context("Observability is required")?; + let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = + if let Some(api) = general_config.api_config.as_ref() { + ( + Some(api.prometheus.listener_port), + Some(api.prometheus.pushgateway_url.clone()), + api.prometheus.push_interval_ms.unwrap_or_default(), + ) + } else { + (None, None, 0) + }; + Ok(Self { + prometheus_port, + prometheus_pushgateway_url, + prometheus_push_interval_ms, + sentry_url: observability.sentry_url.clone(), + sentry_environment: observability.sentry_environment.clone(), + log_format: observability + .log_format + .parse() + .context("Invalid log format")?, + }) + } } diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index 3b33ef43343f..e28d61fa055b 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,6 +213,8 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + pub api_namespaces: Option>, + pub extended_api_tracing: bool, } impl Web3JsonRpcConfig { @@ -251,6 +253,8 @@ impl Web3JsonRpcConfig { mempool_cache_size: Default::default(), tree_api_url: None, whitelisted_tokens_for_aa: Default::default(), + api_namespaces: None, + extended_api_tracing: false, } } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 287a47ce4eaa..a07a0c8972ab 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,10 +1,34 @@ +use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; + use serde::Deserialize; -use zksync_basic_types::{L1ChainId, L2ChainId}; +use zksync_basic_types::{ + commitment::L1BatchCommitmentMode, url::SensitiveUrl, L1ChainId, L2ChainId, +}; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct Pruning { + pub enabled: bool, + pub chunk_size: Option, + pub removal_delay_sec: Option, + pub data_retention_sec: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct SnapshotRecovery { + pub enabled: bool, + pub postgres_max_concurrency: Option, + pub tree_chunk_size: Option, +} /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub main_node_url: String, + pub main_node_url: SensitiveUrl, + pub commitment_generator_max_parallelism: Option, + pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + pub tree_api_remote_url: Option, + pub snapshot_recovery: Option, + pub pruning: Option, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index ad0ef5a4d5b8..1fb10edc0826 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,6 +12,9 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + pub reads_persistence_enabled: bool, + pub processing_delay_ms: u64, + pub include_indices_and_filters_in_block_cache: bool, } impl Default for ExperimentalDBConfig { @@ -20,6 +23,9 @@ impl Default for ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, + reads_persistence_enabled: true, + processing_delay_ms: 0, + include_indices_and_filters_in_block_cache: false, } } } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 55e4d1c82767..b9ed5f272afc 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -97,6 +97,8 @@ impl Distribution for EncodeDist { mempool_cache_update_interval: self.sample(rng), mempool_cache_size: self.sample(rng), whitelisted_tokens_for_aa: self.sample_range(rng).map(|_| rng.gen()).collect(), + api_namespaces: self.sample_range(rng).map(|_| self.sample(rng)).collect(), + extended_api_tracing: self.sample(rng), } } } @@ -280,6 +282,9 @@ impl Distribution for EncodeDist { configs::ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: self.sample(rng), state_keeper_db_max_open_files: self.sample(rng), + reads_persistence_enabled: self.sample(rng), + processing_delay_ms: self.sample(rng), + include_indices_and_filters_in_block_cache: self.sample(rng), } } } diff --git a/core/lib/protobuf_config/src/api.rs b/core/lib/protobuf_config/src/api.rs index fe0cfb3e0d6e..4eac849773f3 100644 --- a/core/lib/protobuf_config/src/api.rs +++ b/core/lib/protobuf_config/src/api.rs @@ -69,7 +69,11 @@ impl ProtoRepr for proto::Web3JsonRpc { }) .collect::>() .context("max_response_body_size_overrides")?; - + let api_namespaces = if self.api_namespaces.is_empty() { + None + } else { + Some(self.api_namespaces.clone()) + }; Ok(Self::Type { http_port: required(&self.http_port) .and_then(|p| Ok((*p).try_into()?)) @@ -154,6 +158,8 @@ impl ProtoRepr for proto::Web3JsonRpc { .map(|(i, k)| parse_h160(k).context(i)) .collect::, _>>() .context("account_pks")?, + extended_api_tracing: self.extended_api_tracing.unwrap_or_default(), + api_namespaces, }) } @@ -222,6 +228,8 @@ impl ProtoRepr for proto::Web3JsonRpc { .iter() .map(|k| format!("{:?}", k)) .collect(), + extended_api_tracing: Some(this.extended_api_tracing), + api_namespaces: this.api_namespaces.clone().unwrap_or_default(), } } } diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 3e90feceb11e..576934aeddfa 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,30 +1,108 @@ +use std::{ + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + str::FromStr, +}; + use anyhow::Context; -use zksync_basic_types::{L1ChainId, L2ChainId}; -use zksync_config::configs::en_config::ENConfig; +use zksync_basic_types::{url::SensitiveUrl, L1ChainId, L2ChainId}; +use zksync_config::configs::en_config::{ENConfig, Pruning, SnapshotRecovery}; use zksync_protobuf::{required, ProtoRepr}; -use crate::proto::en as proto; +use crate::{proto::en as proto, read_optional_repr}; + +impl ProtoRepr for proto::Pruning { + type Type = Pruning; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + chunk_size: self.chunk_size, + removal_delay_sec: self.removal_delay_sec.map(|a| NonZeroU64::new(a)).flatten(), + data_retention_sec: self.data_retention_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + chunk_size: this.chunk_size, + removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), + data_retention_sec: this.data_retention_sec, + } + } +} + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecovery; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + postgres_max_concurrency: self + .postgres_max_concurrency + .map(|a| NonZeroUsize::new(a as usize)) + .flatten(), + tree_chunk_size: self.tree_chunk_size, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), + tree_chunk_size: this.tree_chunk_size, + } + } +} impl ProtoRepr for proto::ExternalNode { type Type = ENConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { - main_node_url: required(&self.main_node_url)?.to_string(), + main_node_url: SensitiveUrl::from_str( + required(&self.main_node_url).context("main_node_url")?, + )?, l1_chain_id: required(&self.l1_chain_id) .map(|x| L1ChainId(*x)) .context("l1_chain_id")?, l2_chain_id: required(&self.l2_chain_id) .and_then(|x| L2ChainId::try_from(*x).map_err(|a| anyhow::anyhow!(a))) .context("l2_chain_id")?, + l1_batch_commit_data_generator_mode: required( + &self.l1_batch_commit_data_generator_mode, + ) + .and_then(|x| Ok(crate::proto::genesis::L1BatchCommitDataGeneratorMode::try_from(*x)?)) + .context("l1_batch_commit_data_generator_mode")? + .parse(), + commitment_generator_max_parallelism: self + .commitment_generator_max_parallelism + .map(NonZeroU32::new) + .flatten(), + tree_api_remote_url: self.tree_api_remote_url.clone(), + pruning: read_optional_repr(&self.pruning).context("pruning")?, + snapshot_recovery: read_optional_repr(&self.snapshot_recovery) + .context("snapshot_recovery")?, }) } fn build(this: &Self::Type) -> Self { Self { - main_node_url: Some(this.main_node_url.clone()), + main_node_url: Some(this.main_node_url.expose_str().to_string()), l1_chain_id: Some(this.l1_chain_id.0), l2_chain_id: Some(this.l2_chain_id.as_u64()), + l1_batch_commit_data_generator_mode: Some( + crate::proto::genesis::L1BatchCommitDataGeneratorMode::new( + &this.l1_batch_commit_data_generator_mode, + ) + .into(), + ), + tree_api_remote_url: this.tree_api_remote_url.clone(), + commitment_generator_max_parallelism: this + .commitment_generator_max_parallelism + .map(|a| a.get()), + snapshot_recovery: None, + pruning: None, } } } diff --git a/core/lib/protobuf_config/src/experimental.rs b/core/lib/protobuf_config/src/experimental.rs index c4fe17aadf43..e3d864c51c42 100644 --- a/core/lib/protobuf_config/src/experimental.rs +++ b/core/lib/protobuf_config/src/experimental.rs @@ -21,6 +21,11 @@ impl ProtoRepr for proto::Db { .map(|count| NonZeroU32::new(count).context("cannot be 0")) .transpose() .context("state_keeper_db_max_open_files")?, + reads_persistence_enabled: self.reads_persistence_enabled.unwrap_or_default(), + processing_delay_ms: self.processing_delay_ms.unwrap_or_default(), + include_indices_and_filters_in_block_cache: self + .include_indices_and_filters_in_block_cache + .unwrap_or_default(), }) } @@ -34,6 +39,11 @@ impl ProtoRepr for proto::Db { state_keeper_db_max_open_files: this .state_keeper_db_max_open_files .map(NonZeroU32::get), + reads_persistence_enabled: Some(this.reads_persistence_enabled), + processing_delay_ms: Some(this.processing_delay_ms), + include_indices_and_filters_in_block_cache: Some( + this.include_indices_and_filters_in_block_cache, + ), } } } diff --git a/core/lib/protobuf_config/src/genesis.rs b/core/lib/protobuf_config/src/genesis.rs index 754f1fc16d0e..a6c23c653ad2 100644 --- a/core/lib/protobuf_config/src/genesis.rs +++ b/core/lib/protobuf_config/src/genesis.rs @@ -11,20 +11,21 @@ use zksync_protobuf::{repr::ProtoRepr, required}; use crate::{parse_h160, parse_h256, proto::genesis as proto}; impl proto::L1BatchCommitDataGeneratorMode { - fn new(n: &L1BatchCommitmentMode) -> Self { + pub(crate) fn new(n: &L1BatchCommitmentMode) -> Self { match n { L1BatchCommitmentMode::Rollup => Self::Rollup, L1BatchCommitmentMode::Validium => Self::Validium, } } - fn parse(&self) -> L1BatchCommitmentMode { + pub(crate) fn parse(&self) -> L1BatchCommitmentMode { match self { Self::Rollup => L1BatchCommitmentMode::Rollup, Self::Validium => L1BatchCommitmentMode::Validium, } } } + impl ProtoRepr for proto::Genesis { type Type = configs::GenesisConfig; fn read(&self) -> anyhow::Result { diff --git a/core/lib/protobuf_config/src/proto/config/api.proto b/core/lib/protobuf_config/src/proto/config/api.proto index 09503056a3f1..d7f0a5326476 100644 --- a/core/lib/protobuf_config/src/proto/config/api.proto +++ b/core/lib/protobuf_config/src/proto/config/api.proto @@ -40,7 +40,8 @@ message Web3JsonRpc { optional uint64 mempool_cache_size = 29; // optional repeated string whitelisted_tokens_for_aa = 30; // optional repeated MaxResponseSizeOverride max_response_body_size_overrides = 31; - + repeated string api_namespaces = 32; + optional bool extended_api_tracing = 33; reserved 15; reserved "l1_to_l2_transactions_compatibility_mode"; } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index ed2553f5aca5..5a4c47f8240d 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -1,10 +1,28 @@ syntax = "proto3"; +import "zksync/config/genesis.proto"; package zksync.config.en; +message SnapshotRecovery { + optional bool enabled = 1; + optional uint64 postgres_max_concurrency = 2; + optional uint64 tree_chunk_size = 3; +} + +message Pruning { + optional bool enabled = 1; + optional uint32 chunk_size = 2; + optional uint64 removal_delay_sec = 3; + optional uint64 data_retention_sec = 4; +} message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required + optional uint32 commitment_generator_max_parallelism = 4; + optional string tree_api_remote_url = 5; + optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 6; // optional, default to rollup + optional SnapshotRecovery snapshot_recovery = 7; + optional Pruning pruning = 8; } diff --git a/core/lib/protobuf_config/src/proto/config/experimental.proto b/core/lib/protobuf_config/src/proto/config/experimental.proto index 4f456b9aca39..ddcb4265319d 100644 --- a/core/lib/protobuf_config/src/proto/config/experimental.proto +++ b/core/lib/protobuf_config/src/proto/config/experimental.proto @@ -8,4 +8,7 @@ package zksync.config.experimental; message DB { optional uint64 state_keeper_db_block_cache_capacity_mb = 1; // MB; required optional uint32 state_keeper_db_max_open_files = 2; // optional + optional bool reads_persistence_enabled = 3; + optional uint64 processing_delay_ms = 4; + optional bool include_indices_and_filters_in_block_cache = 5; } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index cfac1df27cd0..6eecc35da95e 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -1,3 +1,6 @@ +use std::path::PathBuf; + +use anyhow::Context; use zksync_config::{ configs::{ api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, @@ -28,6 +31,12 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { let this: T = zksync_protobuf::serde::deserialize_proto_with_options(d, false)?; this.read() } + +pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { + let yaml = std::fs::read_to_string(&path_buf).context("failed decoding YAML config")?; + decode_yaml_repr::(&yaml) +} + // // TODO (QIT-22): This structure is going to be removed when components will be responsible for their own configs. /// A temporary config store allowing to pass deserialized configs from `zksync_server` to `zksync_core`. diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index a909fdfa9ba0..5a3ff7af5bd6 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,3 +1,4 @@ l1_chain_id: 9 l2_chain_id: 270 -main_node_url: http://localhost:3050 \ No newline at end of file +main_node_url: http://localhost:3050 +l1_batch_commit_data_generator_mode: Rollup \ No newline at end of file diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index d59da18d1266..2b7cfff81a2f 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -128,7 +128,7 @@ eth: aggregated_block_execute_deadline: 10 timestamp_criteria_max_allowed_lag: 30 max_eth_tx_data_size: 120000 - aggregated_proof_sizes: [ 1,4 ] + aggregated_proof_sizes: 1 max_aggregated_tx_gas: 4000000 max_acceptable_priority_fee_in_gwei: 100000000000 pubdata_sending_mode: BLOBS From d6f5535eed27aa67030ca759879f862a68e3f85a Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 10:48:30 +0200 Subject: [PATCH 03/56] Macros Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 67 ++++++++++++++++++------ 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 150c9c4cee5f..6af5bbe54a3b 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -45,6 +45,32 @@ pub(crate) mod observability; #[cfg(test)] mod tests; +macro_rules! load_optional_config_or_default { + ($config:expr, $($name:ident).+, $default:ident) => { + $config + .as_ref() + .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + .unwrap_or_else(OptionalENConfig::$default) + }; +} + +macro_rules! load_config_or_default { + ($config:expr, $($name:ident).+, $default:ident) => { + $config + .as_ref() + .map(|a| a.$($name).+.try_into()).transpose()? + .unwrap_or_else(OptionalENConfig::$default) + }; +} + +macro_rules! load_config { + ($config:expr, $($name:ident).+) => { + $config + .as_ref() + .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + }; +} + const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024; /// Encapsulation of configuration source with a mock implementation used in tests. @@ -444,23 +470,30 @@ impl OptionalENConfig { .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); Ok(OptionalENConfig { - filters_limit: api - .web3_json_rpc - .filters_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_filters_limit), - subscriptions_limit: api - .web3_json_rpc - .subscriptions_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_subscriptions_limit), - req_entities_limit: api - .web3_json_rpc - .req_entities_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_req_entities_limit), - max_tx_size_bytes: api.web3_json_rpc.max_tx_size, - vm_execution_cache_misses_limit: api.web3_json_rpc.vm_execution_cache_misses_limit, + filters_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.filters_limit, + default_filters_limit + ), + subscriptions_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.subscriptions_limit, + default_subscriptions_limit + ), + req_entities_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.req_entities_limit, + default_req_entities_limit + ), + max_tx_size_bytes: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_tx_size, + default_max_tx_size_bytes + ), + vm_execution_cache_misses_limit: load_config!( + general_config.api_config, + web3_json_rpc.vm_execution_cache_misses_limit + ), transactions_per_sec_limit: None, fee_history_limit: api.web3_json_rpc.fee_history_limit(), max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), From 8c035710832028d21836c73d918e40a539b79582 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 11:51:40 +0200 Subject: [PATCH 04/56] More defaults Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 76 +++++++++++++++++++----- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 6af5bbe54a3b..2675f0a96a1c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -58,7 +58,7 @@ macro_rules! load_config_or_default { ($config:expr, $($name:ident).+, $default:ident) => { $config .as_ref() - .map(|a| a.$($name).+.try_into()).transpose()? + .map(|a| a.$($name).+.clone().try_into()).transpose()? .unwrap_or_else(OptionalENConfig::$default) }; } @@ -494,21 +494,63 @@ impl OptionalENConfig { general_config.api_config, web3_json_rpc.vm_execution_cache_misses_limit ), + // Option is deprecated transactions_per_sec_limit: None, - fee_history_limit: api.web3_json_rpc.fee_history_limit(), - max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), - max_response_body_size_mb: api.web3_json_rpc.max_response_body_size().global, - max_response_body_size_overrides_mb: api - .web3_json_rpc - .max_response_body_size() - .overrides, - pubsub_polling_interval_ms: api.web3_json_rpc.pubsub_interval().as_millis() as u64, - max_nonce_ahead: api.web3_json_rpc.max_nonce_ahead, - vm_concurrency_limit: api.web3_json_rpc.vm_concurrency_limit(), - factory_deps_cache_size_mb: api.web3_json_rpc.factory_deps_cache_size(), - initial_writes_cache_size_mb: api.web3_json_rpc.initial_writes_cache_size(), - latest_values_cache_size_mb: api.web3_json_rpc.latest_values_cache_size(), - filters_disabled: api.web3_json_rpc.filters_disabled, + fee_history_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.fee_history_limit, + default_fee_history_limit + ), + max_batch_request_size: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.max_batch_request_size, + default_max_batch_request_size + ), + max_response_body_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.max_response_body_size_mb, + default_max_response_body_size_mb + ), + max_response_body_size_overrides_mb: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_response_body_size_overrides_mb, + default_max_response_body_size_overrides_mb + ), + pubsub_polling_interval_ms: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.pubsub_polling_interval, + default_polling_interval + ), + max_nonce_ahead: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_nonce_ahead, + default_max_nonce_ahead + ), + vm_concurrency_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.vm_concurrency_limit, + default_vm_concurrency_limit + ), + factory_deps_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.factory_deps_cache_size_mb, + default_factory_deps_cache_size_mb + ), + initial_writes_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.initial_writes_cache_size_mb, + default_initial_writes_cache_size_mb + ), + latest_values_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.latest_values_cache_size_mb, + default_latest_values_cache_size_mb + ), + filters_disabled: general_config + .api_config + .as_ref() + .map(|a| a.web3_json_rpc.filters_disabled) + .unwrap_or_default(), mempool_cache_update_interval_ms: api .web3_json_rpc .mempool_cache_update_interval() @@ -671,6 +713,10 @@ impl OptionalENConfig { 10 } + fn default_max_response_body_size_overrides_mb() -> MaxResponseSizeOverrides { + MaxResponseSizeOverrides::empty() + } + const fn default_l2_block_seal_queue_capacity() -> usize { 10 } From c3ca813dd9a1a1ae47483df333263cc05df3b226 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 20:46:29 +0200 Subject: [PATCH 05/56] Use all optional en config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 234 +++++++++++------- core/lib/config/src/configs/en_config.rs | 3 +- core/lib/protobuf_config/src/en.rs | 9 +- .../protobuf_config/src/proto/config/en.proto | 7 +- 4 files changed, 154 insertions(+), 99 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 2675f0a96a1c..54ff79deba6c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -50,7 +50,7 @@ macro_rules! load_optional_config_or_default { $config .as_ref() .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? - .unwrap_or_else(OptionalENConfig::$default) + .unwrap_or_else(Self::$default) }; } @@ -59,7 +59,7 @@ macro_rules! load_config_or_default { $config .as_ref() .map(|a| a.$($name).+.clone().try_into()).transpose()? - .unwrap_or_else(OptionalENConfig::$default) + .unwrap_or_else(Self::$default) }; } @@ -67,7 +67,7 @@ macro_rules! load_config { ($config:expr, $($name:ident).+) => { $config .as_ref() - .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + .map(|a| a.$($name).+.clone().map(|a| a.try_into())).flatten().transpose()? }; } @@ -446,28 +446,8 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { - let api = general_config - .api_config - .as_ref() - .context("Api is required")?; - let db = general_config - .db_config - .as_ref() - .context("Db config is required")?; - let postgres = general_config - .postgres_config - .as_ref() - .context("Db config is required")?; - let state_keeper = general_config - .state_keeper_config - .as_ref() - .context("Db config is required")?; - - let api_namespaces = api - .web3_json_rpc - .api_namespaces - .as_ref() - .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) + .map(|a: Vec| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); Ok(OptionalENConfig { filters_limit: load_optional_config_or_default!( @@ -551,74 +531,138 @@ impl OptionalENConfig { .as_ref() .map(|a| a.web3_json_rpc.filters_disabled) .unwrap_or_default(), - mempool_cache_update_interval_ms: api - .web3_json_rpc - .mempool_cache_update_interval() - .as_millis() as u64, - mempool_cache_size: api.web3_json_rpc.mempool_cache_size(), - - healthcheck_slow_time_limit_ms: api.healthcheck.slow_time_limit_ms, - healthcheck_hard_time_limit_ms: api.healthcheck.hard_time_limit_ms, - estimate_gas_scale_factor: api.web3_json_rpc.gas_price_scale_factor, - estimate_gas_acceptable_overestimation: api - .web3_json_rpc - .estimate_gas_acceptable_overestimation, - gas_price_scale_factor: api.web3_json_rpc.gas_price_scale_factor, - merkle_tree_max_l1_batches_per_iter: db.merkle_tree.max_l1_batches_per_iter, - merkle_tree_max_open_files: db.experimental.state_keeper_db_max_open_files, - merkle_tree_multi_get_chunk_size: db.merkle_tree.multi_get_chunk_size, - merkle_tree_block_cache_size_mb: db - .experimental - .state_keeper_db_block_cache_capacity_mb, - merkle_tree_memtable_capacity_mb: db.merkle_tree.memtable_capacity_mb, - merkle_tree_stalled_writes_timeout_sec: db.merkle_tree.stalled_writes_timeout_sec, - database_long_connection_threshold_ms: postgres.long_connection_threshold_ms, - database_slow_query_threshold_ms: postgres.slow_query_threshold_ms, - l2_block_seal_queue_capacity: state_keeper.l2_block_seal_queue_capacity, - main_node_rate_limit_rps: Self::default_main_node_rate_limit_rps(), + mempool_cache_update_interval_ms: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.mempool_cache_update_interval, + default_mempool_cache_update_interval_ms + ), + mempool_cache_size: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.mempool_cache_size, + default_mempool_cache_size + ), + + healthcheck_slow_time_limit_ms: load_config!( + general_config.api_config, + healthcheck.slow_time_limit_ms + ), + healthcheck_hard_time_limit_ms: load_config!( + general_config.api_config, + healthcheck.hard_time_limit_ms + ), + estimate_gas_scale_factor: load_config_or_default!( + general_config.api_config, + web3_json_rpc.estimate_gas_scale_factor, + default_estimate_gas_scale_factor + ), + estimate_gas_acceptable_overestimation: load_config_or_default!( + general_config.api_config, + web3_json_rpc.estimate_gas_acceptable_overestimation, + default_estimate_gas_acceptable_overestimation + ), + gas_price_scale_factor: load_config_or_default!( + general_config.api_config, + web3_json_rpc.gas_price_scale_factor, + default_gas_price_scale_factor + ), + merkle_tree_max_l1_batches_per_iter: load_config_or_default!( + general_config.db_config, + merkle_tree.max_l1_batches_per_iter, + default_merkle_tree_max_l1_batches_per_iter + ), + merkle_tree_max_open_files: load_config!( + general_config.db_config, + experimental.state_keeper_db_max_open_files + ), + merkle_tree_multi_get_chunk_size: load_config_or_default!( + general_config.db_config, + merkle_tree.multi_get_chunk_size, + default_merkle_tree_multi_get_chunk_size + ), + merkle_tree_block_cache_size_mb: load_config_or_default!( + general_config.db_config, + experimental.state_keeper_db_block_cache_capacity_mb, + default_merkle_tree_block_cache_size_mb + ), + merkle_tree_memtable_capacity_mb: load_config_or_default!( + general_config.db_config, + merkle_tree.memtable_capacity_mb, + default_merkle_tree_memtable_capacity_mb + ), + merkle_tree_stalled_writes_timeout_sec: load_config_or_default!( + general_config.db_config, + merkle_tree.stalled_writes_timeout_sec, + default_merkle_tree_stalled_writes_timeout_sec + ), + database_long_connection_threshold_ms: load_config!( + general_config.postgres_config, + long_connection_threshold_ms + ), + database_slow_query_threshold_ms: load_config!( + general_config.postgres_config, + slow_query_threshold_ms + ), + l2_block_seal_queue_capacity: load_config_or_default!( + general_config.state_keeper_config, + l2_block_seal_queue_capacity, + default_l2_block_seal_queue_capacity + ), l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, snapshots_recovery_enabled: enconfig .snapshot_recovery .as_ref() .map(|a| a.enabled) .unwrap_or_default(), - snapshots_recovery_postgres_max_concurrency: enconfig - .snapshot_recovery - .as_ref() - .map(|a| a.postgres_max_concurrency) - .flatten() - .unwrap_or_else(Self::default_snapshots_recovery_postgres_max_concurrency), + snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( + enconfig.snapshot_recovery, + postgres_max_concurrency, + default_snapshots_recovery_postgres_max_concurrency + ), pruning_enabled: enconfig .pruning .as_ref() .map(|a| a.enabled) .unwrap_or_default(), - pruning_chunk_size: enconfig - .pruning - .as_ref() - .map(|a| a.chunk_size) - .flatten() - .unwrap_or_else(Self::default_pruning_chunk_size), - pruning_removal_delay_sec: enconfig - .pruning + pruning_chunk_size: load_optional_config_or_default!( + enconfig.pruning, + chunk_size, + default_pruning_chunk_size + ), + pruning_removal_delay_sec: load_optional_config_or_default!( + enconfig.pruning, + removal_delay_sec, + default_pruning_removal_delay_sec + ), + pruning_data_retention_sec: load_optional_config_or_default!( + enconfig.pruning, + data_retention_sec, + default_pruning_data_retention_sec + ), + protective_reads_persistence_enabled: general_config + .db_config .as_ref() - .map(|a| a.removal_delay_sec) - .flatten() - .unwrap_or_else(Self::default_pruning_removal_delay_sec), - pruning_data_retention_sec: enconfig - .pruning + .map(|a| a.experimental.reads_persistence_enabled) + .unwrap_or_default(), + merkle_tree_processing_delay_ms: load_config_or_default!( + general_config.db_config, + experimental.processing_delay_ms, + default_merkle_tree_processing_delay_ms + ), + merkle_tree_include_indices_and_filters_in_block_cache: general_config + .db_config .as_ref() - .map(|a| a.data_retention_sec) - .flatten() - .unwrap_or_else(Self::default_pruning_data_retention_sec), - contracts_diamond_proxy_addr: None, - protective_reads_persistence_enabled: db.experimental.reads_persistence_enabled, - merkle_tree_processing_delay_ms: db.experimental.processing_delay_ms, - merkle_tree_include_indices_and_filters_in_block_cache: db - .experimental - .include_indices_and_filters_in_block_cache, + .map(|a| a.experimental.include_indices_and_filters_in_block_cache) + .unwrap_or_default(), + extended_rpc_tracing: load_config_or_default!( + general_config.api_config, + web3_json_rpc.extended_api_tracing, + default_extended_api_tracing + ), + main_node_rate_limit_rps: enconfig + .main_node_rate_limit_rps + .unwrap_or_else(Self::default_main_node_rate_limit_rps), api_namespaces, - extended_rpc_tracing: api.web3_json_rpc.extended_api_tracing, + contracts_diamond_proxy_addr: None, }) } @@ -1027,22 +1071,26 @@ impl ExperimentalENConfig { } pub fn from_configs( general_config: &GeneralConfig, - enconfig: &ENConfig, + external_node_config: &ENConfig, ) -> anyhow::Result { - let db_config = general_config.db_config.as_ref().context("db_config")?; - let snapshot_recovery = enconfig - .snapshot_recovery - .as_ref() - .context("snapshot_recovery")?; Ok(Self { - state_keeper_db_block_cache_capacity_mb: db_config - .experimental - .state_keeper_db_block_cache_capacity_mb, - state_keeper_db_max_open_files: db_config.experimental.state_keeper_db_max_open_files, - snapshots_recovery_tree_chunk_size: snapshot_recovery - .tree_chunk_size - .unwrap_or_else(Self::default_snapshots_recovery_tree_chunk_size), - commitment_generator_max_parallelism: enconfig.commitment_generator_max_parallelism, + state_keeper_db_block_cache_capacity_mb: load_config_or_default!( + general_config.db_config, + experimental.state_keeper_db_block_cache_capacity_mb, + default_state_keeper_db_block_cache_capacity_mb + ), + + state_keeper_db_max_open_files: load_config!( + general_config.db_config, + experimental.state_keeper_db_max_open_files + ), + snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( + external_node_config.snapshot_recovery, + tree_chunk_size, + default_snapshots_recovery_tree_chunk_size + ), + commitment_generator_max_parallelism: external_node_config + .commitment_generator_max_parallelism, }) } } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index a07a0c8972ab..20e8a9fc9558 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -26,8 +26,9 @@ pub struct ENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, pub main_node_url: SensitiveUrl, - pub commitment_generator_max_parallelism: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + pub main_node_rate_limit_rps: Option, + pub commitment_generator_max_parallelism: Option, pub tree_api_remote_url: Option, pub snapshot_recovery: Option, pub pruning: Option, diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 576934aeddfa..67a70a3ba7fb 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -80,6 +80,10 @@ impl ProtoRepr for proto::ExternalNode { .map(NonZeroU32::new) .flatten(), tree_api_remote_url: self.tree_api_remote_url.clone(), + main_node_rate_limit_rps: self + .main_node_rate_limit_rps + .map(|a| NonZeroUsize::new(a as usize)) + .flatten(), pruning: read_optional_repr(&self.pruning).context("pruning")?, snapshot_recovery: read_optional_repr(&self.snapshot_recovery) .context("snapshot_recovery")?, @@ -101,8 +105,9 @@ impl ProtoRepr for proto::ExternalNode { commitment_generator_max_parallelism: this .commitment_generator_max_parallelism .map(|a| a.get()), - snapshot_recovery: None, - pruning: None, + main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), + snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), + pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 5a4c47f8240d..45a4d68f12de 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -22,7 +22,8 @@ message ExternalNode { optional uint64 l1_chain_id = 3; // required optional uint32 commitment_generator_max_parallelism = 4; optional string tree_api_remote_url = 5; - optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 6; // optional, default to rollup - optional SnapshotRecovery snapshot_recovery = 7; - optional Pruning pruning = 8; + optional uint32 main_node_rate_limit_rps = 6; + optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup + optional SnapshotRecovery snapshot_recovery = 8; + optional Pruning pruning = 9; } From 78124cb4272fc5186e9539102c77fe8b5c405767 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 21:11:52 +0200 Subject: [PATCH 06/56] Allowing to run external node from files Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 - .../external_node/src/config/observability.rs | 35 +++++++++++++------ core/bin/external_node/src/main.rs | 33 ++++++++++++++++- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 54ff79deba6c..0b482fa84cea 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -7,7 +7,6 @@ use std::{ }; use anyhow::Context; -use futures::StreamExt; use serde::Deserialize; use zksync_config::{ configs::{ diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index eec8bf2d76a7..a4aa9970ca0d 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -4,7 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use serde::Deserialize; use vlog::LogFormat; -use zksync_config::configs::{GeneralConfig, ObservabilityConfig}; +use zksync_config::configs::GeneralConfig; use super::{ConfigurationSource, Environment}; @@ -27,6 +27,8 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, + /// Log directives, same as RUST_LOG + pub log_directives: Option, } impl ObservabilityENConfig { @@ -81,6 +83,9 @@ impl ObservabilityENConfig { pub fn build_observability(&self) -> anyhow::Result { let mut builder = vlog::ObservabilityBuilder::new().with_log_format(self.log_format); + if let Some(log_directives) = &self.log_directives { + builder = builder.with_log_directives(log_directives.clone()) + } // Some legacy deployments use `unset` as an equivalent of `None`. let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset"); if let Some(sentry_url) = sentry_url { @@ -101,10 +106,20 @@ impl ObservabilityENConfig { } pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { - let observability = general_config - .observability - .as_ref() - .context("Observability is required")?; + let (sentry_url, sentry_environment, log_format, log_directives) = + if let Some(observability) = general_config.observability.as_ref() { + ( + observability.sentry_url.clone(), + observability.sentry_environment.clone(), + observability + .log_format + .parse() + .context("Invalid log format")?, + observability.log_directives.clone(), + ) + } else { + (None, None, LogFormat::default(), None) + }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = if let Some(api) = general_config.api_config.as_ref() { ( @@ -119,12 +134,10 @@ impl ObservabilityENConfig { prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms, - sentry_url: observability.sentry_url.clone(), - sentry_environment: observability.sentry_environment.clone(), - log_format: observability - .log_format - .parse() - .context("Invalid log format")?, + sentry_url, + sentry_environment, + log_format, + log_directives, }) } } diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 0f53e8983881..431cca1d6d70 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -707,6 +707,18 @@ struct Cli { /// Comma-separated list of components to launch. #[arg(long, default_value = "all")] components: ComponentsToRun, + /// Path to the yaml config. If set, it will be used instead of env vars. + #[arg(long)] + config_path: Option, + /// Path to the yaml with secrets. If set, it will be used instead of env vars. + #[arg(long)] + secrets_path: Option, + /// Path to the yaml with genesis. If set, it will be used instead of env vars. + #[arg(long)] + external_node_config_path: Option, + /// Path to the yaml with genesis. If set, it will be used instead of env vars. + #[arg(long)] + consensus_path: Option, } #[derive(Debug, Clone, Copy, PartialEq, Hash, Eq)] @@ -763,7 +775,26 @@ async fn main() -> anyhow::Result<()> { // Initial setup. let opt = Cli::parse(); - let mut config = ExternalNodeConfig::new().context("Failed to load node configuration")?; + let mut config = if let Some(config_path) = opt.config_path.clone() { + let secrets_path = opt + .secrets_path + .clone() + .context("File based config can't be used partially please provide all paths")?; + let external_node_config_path = opt + .external_node_config_path + .clone() + .context("File based config can't be used partially please provide all paths")?; + + ExternalNodeConfig::from_files( + config_path, + external_node_config_path, + secrets_path, + opt.consensus_path.clone(), + )? + } else { + ExternalNodeConfig::new().context("Failed to load node configuration")? + }; + if !opt.enable_consensus { config.consensus = None; } From da675b5757682c3048c9c27a4da4be439973a37a Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 12:30:21 +0200 Subject: [PATCH 07/56] Fix lints Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 2 +- core/bin/external_node/src/tests.rs | 8 ++++++++ core/lib/env_config/src/api.rs | 4 ++++ core/lib/protobuf_config/src/en.rs | 11 ++++------- .../src/temp_config_store/mod.rs | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 0b482fa84cea..e918c96ff50e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1215,7 +1215,7 @@ impl ExternalNodeConfig<()> { .context("failed decoding secrets YAML config")?; let consensus = consensus - .map(|path| read_yaml_repr::(path)) + .map(read_yaml_repr::) .transpose() .context("failed decoding consensus YAML config")?; diff --git a/core/bin/external_node/src/tests.rs b/core/bin/external_node/src/tests.rs index 00301e1b8234..5974e312d434 100644 --- a/core/bin/external_node/src/tests.rs +++ b/core/bin/external_node/src/tests.rs @@ -156,6 +156,10 @@ async fn external_node_basics(components_str: &'static str) { revert_pending_l1_batch: false, enable_consensus: false, components, + config_path: None, + secrets_path: None, + external_node_config_path: None, + consensus_path: None, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { @@ -265,6 +269,10 @@ async fn node_reacts_to_stop_signal_during_initial_reorg_detection() { revert_pending_l1_batch: false, enable_consensus: false, components: "core".parse().unwrap(), + config_path: None, + secrets_path: None, + external_node_config_path: None, + consensus_path: None, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { diff --git a/core/lib/env_config/src/api.rs b/core/lib/env_config/src/api.rs index 6f1948241c9e..68af37393bba 100644 --- a/core/lib/env_config/src/api.rs +++ b/core/lib/env_config/src/api.rs @@ -98,6 +98,8 @@ mod tests { addr("0x0000000000000000000000000000000000000001"), addr("0x0000000000000000000000000000000000000002"), ], + api_namespaces: Some(vec!["debug".to_string()]), + extended_api_tracing: true, }, prometheus: PrometheusConfig { listener_port: 3312, @@ -129,6 +131,8 @@ mod tests { API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 + API_WEB3_JSON_RPC_API_NAMESPACES=debug + API_WEB3_JSON_RPC_EXTENDED_API_TRACING=true API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" API_WEB3_JSON_RPC_WHITELISTED_TOKENS_FOR_AA="0x0000000000000000000000000000000000000001,0x0000000000000000000000000000000000000002" API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 67a70a3ba7fb..8e3b99d1aaa8 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -17,7 +17,7 @@ impl ProtoRepr for proto::Pruning { Ok(Self::Type { enabled: self.enabled.unwrap_or_default(), chunk_size: self.chunk_size, - removal_delay_sec: self.removal_delay_sec.map(|a| NonZeroU64::new(a)).flatten(), + removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), data_retention_sec: self.data_retention_sec, }) } @@ -40,8 +40,7 @@ impl ProtoRepr for proto::SnapshotRecovery { enabled: self.enabled.unwrap_or_default(), postgres_max_concurrency: self .postgres_max_concurrency - .map(|a| NonZeroUsize::new(a as usize)) - .flatten(), + .and_then(|a| NonZeroUsize::new(a as usize)), tree_chunk_size: self.tree_chunk_size, }) } @@ -77,13 +76,11 @@ impl ProtoRepr for proto::ExternalNode { .parse(), commitment_generator_max_parallelism: self .commitment_generator_max_parallelism - .map(NonZeroU32::new) - .flatten(), + .and_then(NonZeroU32::new), tree_api_remote_url: self.tree_api_remote_url.clone(), main_node_rate_limit_rps: self .main_node_rate_limit_rps - .map(|a| NonZeroUsize::new(a as usize)) - .flatten(), + .and_then(|a| NonZeroUsize::new(a as usize)), pruning: read_optional_repr(&self.pruning).context("pruning")?, snapshot_recovery: read_optional_repr(&self.snapshot_recovery) .context("snapshot_recovery")?, diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 6eecc35da95e..a2ea9cb8afe4 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -33,7 +33,7 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { } pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { - let yaml = std::fs::read_to_string(&path_buf).context("failed decoding YAML config")?; + let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; decode_yaml_repr::(&yaml) } From 3c17e1accf9c0bb6030b314f4c04752e829a7922 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:25:45 +0200 Subject: [PATCH 08/56] Add default for new envs Signed-off-by: Danil --- core/lib/config/src/configs/api.rs | 2 ++ core/lib/config/src/configs/experimental.rs | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index e28d61fa055b..7616ee213cbc 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,7 +213,9 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + #[serde(default)] pub api_namespaces: Option>, + #[serde(default)] pub extended_api_tracing: bool, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 1fb10edc0826..833bc4556948 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,8 +12,11 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + #[serde(default = "ExperimentalDBConfig::default_protective_reads_persistence_enabled")] pub reads_persistence_enabled: bool, + #[serde(default = "ExperimentalDBConfig::default_merkle_tree_processing_delay_ms")] pub processing_delay_ms: u64, + #[serde(default)] pub include_indices_and_filters_in_block_cache: bool, } @@ -23,7 +26,7 @@ impl Default for ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, - reads_persistence_enabled: true, + reads_persistence_enabled: Self::default_protective_reads_persistence_enabled(), processing_delay_ms: 0, include_indices_and_filters_in_block_cache: false, } @@ -38,4 +41,10 @@ impl ExperimentalDBConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * super::BYTES_IN_MEGABYTE } + const fn default_protective_reads_persistence_enabled() -> bool { + true + } + const fn default_merkle_tree_processing_delay_ms() -> u64 { + 100 + } } From f7e65e4a6539d48e1435ef2f349c5fcac89710d0 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:42:49 +0200 Subject: [PATCH 09/56] Add more comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 + core/bin/external_node/src/main.rs | 2 +- core/lib/config/src/configs/api.rs | 3 +++ core/lib/config/src/configs/en_config.rs | 7 ++++++- core/lib/config/src/configs/experimental.rs | 9 +++++++++ core/lib/protobuf_config/src/proto/config/api.proto | 4 ++-- core/lib/protobuf_config/src/proto/config/en.proto | 6 +++--- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e918c96ff50e..f5a15da2c616 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1068,6 +1068,7 @@ impl ExperimentalENConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } + pub fn from_configs( general_config: &GeneralConfig, external_node_config: &ENConfig, diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 431cca1d6d70..efa4d5da2209 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -716,7 +716,7 @@ struct Cli { /// Path to the yaml with genesis. If set, it will be used instead of env vars. #[arg(long)] external_node_config_path: Option, - /// Path to the yaml with genesis. If set, it will be used instead of env vars. + /// Path to the yaml with consensus. #[arg(long)] consensus_path: Option, } diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index 7616ee213cbc..e039ab10116a 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,8 +213,11 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + /// Enabled JSON RPC API namespaces. If not set, all namespaces will be available #[serde(default)] pub api_namespaces: Option>, + /// Enables extended tracing of RPC calls. This may negatively impact performance for nodes under high load + /// (hundreds or thousands RPS). #[serde(default)] pub extended_api_tracing: bool, } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 20e8a9fc9558..ab8e49617e84 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -23,11 +23,16 @@ pub struct SnapshotRecovery { /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { + // Genesis pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub main_node_url: SensitiveUrl, pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + + // Main node configuration + pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, + + // En specific components pub commitment_generator_max_parallelism: Option, pub tree_api_remote_url: Option, pub snapshot_recovery: Option, diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 833bc4556948..31bdf3a4730f 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,10 +12,19 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + /// Configures whether to persist protective reads when persisting L1 batches in the state keeper. + /// Protective reads are never required by full nodes so far, not until such a node runs a full Merkle tree + /// (presumably, to participate in L1 batch proving). + /// By default, set to `true` as a temporary safety measure. #[serde(default = "ExperimentalDBConfig::default_protective_reads_persistence_enabled")] pub reads_persistence_enabled: bool, + // Merkle tree config + /// Processing delay between processing L1 batches in the Merkle tree. #[serde(default = "ExperimentalDBConfig::default_merkle_tree_processing_delay_ms")] pub processing_delay_ms: u64, + /// If specified, RocksDB indices and Bloom filters will be managed by the block cache, rather than + /// being loaded entirely into RAM on the RocksDB initialization. The block cache capacity should be increased + /// correspondingly; otherwise, RocksDB performance can significantly degrade. #[serde(default)] pub include_indices_and_filters_in_block_cache: bool, } diff --git a/core/lib/protobuf_config/src/proto/config/api.proto b/core/lib/protobuf_config/src/proto/config/api.proto index d7f0a5326476..4fea0691f79d 100644 --- a/core/lib/protobuf_config/src/proto/config/api.proto +++ b/core/lib/protobuf_config/src/proto/config/api.proto @@ -40,8 +40,8 @@ message Web3JsonRpc { optional uint64 mempool_cache_size = 29; // optional repeated string whitelisted_tokens_for_aa = 30; // optional repeated MaxResponseSizeOverride max_response_body_size_overrides = 31; - repeated string api_namespaces = 32; - optional bool extended_api_tracing = 33; + repeated string api_namespaces = 32; // Optional, if empty all namespaces are available + optional bool extended_api_tracing = 33; // optional, default false reserved 15; reserved "l1_to_l2_transactions_compatibility_mode"; } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 45a4d68f12de..375872d36df4 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -20,9 +20,9 @@ message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required - optional uint32 commitment_generator_max_parallelism = 4; - optional string tree_api_remote_url = 5; - optional uint32 main_node_rate_limit_rps = 6; + optional uint32 commitment_generator_max_parallelism = 4; // optional + optional string tree_api_remote_url = 5; // optional + optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup optional SnapshotRecovery snapshot_recovery = 8; optional Pruning pruning = 9; From 3e5d6c7bfbca0321fa7aa6cccf7597541d1c3d86 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:42:49 +0200 Subject: [PATCH 10/56] Add more comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 8 ++++++-- core/bin/zksync_server/src/node_builder.rs | 8 +++++++- etc/env/file_based/external_node.yaml | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index f5a15da2c616..9f807b73a79e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -446,7 +446,11 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + .map(|a: Vec| { + let result: Result, _> = a.iter().map(|a| serde_json::from_str(a)).collect(); + result + }) + .transpose()?; Ok(OptionalENConfig { filters_limit: load_optional_config_or_default!( @@ -949,7 +953,7 @@ impl RequiredENConfig { eth_client_url: secrets .l1 .as_ref() - .context("Eth config is required")? + .context("L1 secrets are required")? .l1_rpc_url .clone(), main_node_url: en_config.main_node_url.clone(), diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 163835044cac..803d540621dc 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -293,7 +293,13 @@ impl MainNodeBuilder { let circuit_breaker_config = try_load_config!(self.configs.circuit_breaker_config); let with_debug_namespace = state_keeper_config.save_call_traces; - let mut namespaces = Namespace::DEFAULT.to_vec(); + let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { + let result: Result, _> = + namespaces.iter().map(|a| serde_json::from_str(a)).collect(); + result? + } else { + Namespace::DEFAULT.to_vec() + }; if with_debug_namespace { namespaces.push(Namespace::Debug) } diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index 5a3ff7af5bd6..6757ad10526d 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,4 +1,4 @@ l1_chain_id: 9 l2_chain_id: 270 main_node_url: http://localhost:3050 -l1_batch_commit_data_generator_mode: Rollup \ No newline at end of file +l1_batch_commit_data_generator_mode: Rollup From 551dcb7cdaf185a72fb6b01233eef64d81c147c6 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 15:29:12 +0200 Subject: [PATCH 11/56] Support extended_rpc_tracing Signed-off-by: Danil --- core/bin/zksync_server/src/node_builder.rs | 1 + core/node/node_framework/examples/main_node.rs | 1 + .../src/implementations/layers/web3_api/server.rs | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 803d540621dc..e50b8ba03600 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -315,6 +315,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + extended_rpc_tracing: Some(rpc_config.extended_api_tracing), }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, diff --git a/core/node/node_framework/examples/main_node.rs b/core/node/node_framework/examples/main_node.rs index f42cf76d33a2..813e655267d1 100644 --- a/core/node/node_framework/examples/main_node.rs +++ b/core/node/node_framework/examples/main_node.rs @@ -286,6 +286,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + extended_rpc_tracing: Some(rpc_config.extended_api_tracing), }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server.rs b/core/node/node_framework/src/implementations/layers/web3_api/server.rs index 08eaa4b80444..18022b088658 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/server.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/server.rs @@ -29,6 +29,7 @@ pub struct Web3ServerOptionalConfig { pub websocket_requests_per_minute_limit: Option, // used by circuit breaker. pub replication_lag_limit: Option, + pub extended_rpc_tracing: Option, } impl Web3ServerOptionalConfig { @@ -53,6 +54,9 @@ impl Web3ServerOptionalConfig { api_builder = api_builder .with_websocket_requests_per_minute_limit(websocket_requests_per_minute_limit); } + if let Some(extended_rpc_tracing) = self.extended_rpc_tracing { + api_builder = api_builder.with_extended_tracing(extended_rpc_tracing) + } api_builder } } From 69e99d8dfc5241f4e46476842d2d8d0afb219a83 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 16:32:55 +0200 Subject: [PATCH 12/56] Update configs Signed-off-by: Danil --- etc/env/file_based/external_node.yaml | 14 +++++++++++++- etc/env/file_based/general.yaml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index 6757ad10526d..edb2b4af597f 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,4 +1,16 @@ l1_chain_id: 9 l2_chain_id: 270 -main_node_url: http://localhost:3050 l1_batch_commit_data_generator_mode: Rollup +commitment_generator_max_parallelism: 10 + +main_node_url: http://localhost:3050 +main_node_rate_limit_rps: 1000 +snapshot_recovery: + enabled: true + postgres_max_concurrency: 10 + tree_chunk_size: 200000 +pruning: + enabled: true + chunk_size: 10 + removal_delay_sec: 60 + data_retention_sec: 3600 diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 2b7cfff81a2f..7ed53bf8fed1 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -128,7 +128,7 @@ eth: aggregated_block_execute_deadline: 10 timestamp_criteria_max_allowed_lag: 30 max_eth_tx_data_size: 120000 - aggregated_proof_sizes: 1 + aggregated_proof_sizes: [ 1 ] max_aggregated_tx_gas: 4000000 max_acceptable_priority_fee_in_gwei: 100000000000 pubdata_sending_mode: BLOBS From 116e06b3868474329d61ce03945d384c2babda72 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 16:32:55 +0200 Subject: [PATCH 13/56] Update configs Signed-off-by: Danil --- core/bin/external_node/src/config/observability.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index a4aa9970ca0d..efec5a8a0d87 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -27,7 +27,7 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, - /// Log directives, same as RUST_LOG + // Log directives in format that is used in `RUST_LOG` pub log_directives: Option, } From f8bdb818857031f78dd36fb996b8539b2e0d03e1 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 17:45:49 +0200 Subject: [PATCH 14/56] Fix tests Signed-off-by: Danil --- core/lib/config/src/testonly.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index b9ed5f272afc..3c6d8d8031ea 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -97,7 +97,8 @@ impl Distribution for EncodeDist { mempool_cache_update_interval: self.sample(rng), mempool_cache_size: self.sample(rng), whitelisted_tokens_for_aa: self.sample_range(rng).map(|_| rng.gen()).collect(), - api_namespaces: self.sample_range(rng).map(|_| self.sample(rng)).collect(), + api_namespaces: self + .sample_opt(|| self.sample_range(rng).map(|_| self.sample(rng)).collect()), extended_api_tracing: self.sample(rng), } } From 92036975c94babefe2dccd98eea9eb08a1846e90 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 4 Jun 2024 10:41:20 +0200 Subject: [PATCH 15/56] Update contract submodule --- contracts | 2 +- core/lib/contracts/src/lib.rs | 2 +- zk_toolbox/crates/config/src/consts.rs | 2 +- .../config/src/forge_interface/script_params.rs | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contracts b/contracts index 32ca4e665da8..8a70bbbc4812 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 32ca4e665da89f5b4f2f705eee40d91024ad5b48 +Subproject commit 8a70bbbc48125f5bde6189b4e3c6a3ee79631678 diff --git a/core/lib/contracts/src/lib.rs b/core/lib/contracts/src/lib.rs index e27728272150..50fc20c5916f 100644 --- a/core/lib/contracts/src/lib.rs +++ b/core/lib/contracts/src/lib.rs @@ -30,7 +30,7 @@ pub enum ContractLanguage { /// Meanwhile, hardhat has one more intermediate folder. That's why, we have to represent each contract /// by two constants, intermediate folder and actual contract name. For Forge we use only second part const HARDHAT_PATH_PREFIX: &str = "contracts/l1-contracts/artifacts/contracts"; -const FORGE_PATH_PREFIX: &str = "contracts/l1-contracts-foundry/out"; +const FORGE_PATH_PREFIX: &str = "contracts/l1-contracts/out"; const BRIDGEHUB_CONTRACT_FILE: (&str, &str) = ("bridgehub", "IBridgehub.sol/IBridgehub.json"); const STATE_TRANSITION_CONTRACT_FILE: (&str, &str) = ( diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 9082a17abb24..90645ff19acf 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -31,7 +31,7 @@ pub(crate) const LOCAL_DB_PATH: &str = "db/"; pub(crate) const ECOSYSTEM_PATH: &str = "etc/ecosystem"; /// Path to l1 contracts foundry folder inside zksync-era -pub(crate) const L1_CONTRACTS_FOUNDRY: &str = "contracts/l1-contracts-foundry"; +pub(crate) const L1_CONTRACTS_FOUNDRY: &str = "contracts/l1-contracts"; pub(crate) const ERA_CHAIN_ID: ChainId = ChainId(270); diff --git a/zk_toolbox/crates/config/src/forge_interface/script_params.rs b/zk_toolbox/crates/config/src/forge_interface/script_params.rs index a01a15be2a01..0e27871b6708 100644 --- a/zk_toolbox/crates/config/src/forge_interface/script_params.rs +++ b/zk_toolbox/crates/config/src/forge_interface/script_params.rs @@ -29,35 +29,35 @@ impl ForgeScriptParams { pub const DEPLOY_ECOSYSTEM_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-l1.toml", output: "script-out/output-deploy-l1.toml", - script_path: "script/DeployL1.s.sol", + script_path: "deploy_scripts/DeployL1.s.sol", }; pub const INITIALIZE_BRIDGES_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-initialize-shared-bridges.toml", output: "script-out/output-initialize-shared-bridges.toml", - script_path: "script/InitializeSharedBridgeOnL2.sol", + script_path: "deploy_scripts/InitializeSharedBridgeOnL2.sol", }; pub const REGISTER_CHAIN_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/register-hyperchain.toml", output: "script-out/output-register-hyperchain.toml", - script_path: "script/RegisterHyperchain.s.sol", + script_path: "deploy_scripts/RegisterHyperchain.s.sol", }; pub const DEPLOY_ERC20_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-erc20.toml", output: "script-out/output-deploy-erc20.toml", - script_path: "script/DeployErc20.s.sol", + script_path: "deploy_scripts/DeployErc20.s.sol", }; pub const DEPLOY_PAYMASTER_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-paymaster.toml", output: "script-out/output-deploy-paymaster.toml", - script_path: "script/DeployPaymaster.s.sol", + script_path: "deploy_scripts/DeployPaymaster.s.sol", }; pub const ACCEPT_GOVERNANCE_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-accept-admin.toml", output: "script-out/output-accept-admin.toml", - script_path: "script/AcceptAdmin.s.sol", + script_path: "deploy_scripts/AcceptAdmin.s.sol", }; From b2230c1b0a61d60d70a9b6b94f0482318995a707 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 4 Jun 2024 10:42:21 +0200 Subject: [PATCH 16/56] Fix script path --- .../config/src/forge_interface/script_params.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zk_toolbox/crates/config/src/forge_interface/script_params.rs b/zk_toolbox/crates/config/src/forge_interface/script_params.rs index 0e27871b6708..70ed08ec5651 100644 --- a/zk_toolbox/crates/config/src/forge_interface/script_params.rs +++ b/zk_toolbox/crates/config/src/forge_interface/script_params.rs @@ -29,35 +29,35 @@ impl ForgeScriptParams { pub const DEPLOY_ECOSYSTEM_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-l1.toml", output: "script-out/output-deploy-l1.toml", - script_path: "deploy_scripts/DeployL1.s.sol", + script_path: "deploy-scripts/DeployL1.s.sol", }; pub const INITIALIZE_BRIDGES_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-initialize-shared-bridges.toml", output: "script-out/output-initialize-shared-bridges.toml", - script_path: "deploy_scripts/InitializeSharedBridgeOnL2.sol", + script_path: "deploy-scripts/InitializeSharedBridgeOnL2.sol", }; pub const REGISTER_CHAIN_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/register-hyperchain.toml", output: "script-out/output-register-hyperchain.toml", - script_path: "deploy_scripts/RegisterHyperchain.s.sol", + script_path: "deploy-scripts/RegisterHyperchain.s.sol", }; pub const DEPLOY_ERC20_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-erc20.toml", output: "script-out/output-deploy-erc20.toml", - script_path: "deploy_scripts/DeployErc20.s.sol", + script_path: "deploy-scripts/DeployErc20.s.sol", }; pub const DEPLOY_PAYMASTER_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-paymaster.toml", output: "script-out/output-deploy-paymaster.toml", - script_path: "deploy_scripts/DeployPaymaster.s.sol", + script_path: "deploy-scripts/DeployPaymaster.s.sol", }; pub const ACCEPT_GOVERNANCE_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-accept-admin.toml", output: "script-out/output-accept-admin.toml", - script_path: "deploy_scripts/AcceptAdmin.s.sol", + script_path: "deploy-scripts/AcceptAdmin.s.sol", }; From 5837cf0b202b64465690b228065b04f946ee6238 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 13:54:08 +0200 Subject: [PATCH 17/56] Set requires between configs Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 12 ++++++------ core/bin/external_node/src/main.rs | 18 +++++------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index a588813522ae..62f0b34e5a58 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1203,19 +1203,19 @@ impl ExternalNodeConfig<()> { pub fn from_files( general_config_path: PathBuf, - external_node_config_patch: PathBuf, - secrets_configs: PathBuf, - consensus: Option, + external_node_config_path: PathBuf, + secrets_configs_path: PathBuf, + consensus_config_path: Option, ) -> anyhow::Result { let general_config = read_yaml_repr::(general_config_path) .context("failed decoding general YAML config")?; let external_node_config = - read_yaml_repr::(external_node_config_patch) + read_yaml_repr::(external_node_config_path) .context("failed decoding external node YAML config")?; - let secrets_config = read_yaml_repr::(secrets_configs) + let secrets_config = read_yaml_repr::(secrets_configs_path) .context("failed decoding secrets YAML config")?; - let consensus = consensus + let consensus = consensus_config_path .map(read_yaml_repr::) .transpose() .context("failed decoding consensus YAML config")?; diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 2ace1aad8681..cd400f9cdcc5 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -701,16 +701,15 @@ struct Cli { #[arg(long, default_value = "all")] components: ComponentsToRun, /// Path to the yaml config. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "secrets_path")] config_path: Option, /// Path to the yaml with secrets. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "external_node_config_path")] secrets_path: Option, /// Path to the yaml with genesis. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "config_path")] external_node_config_path: Option, /// Path to the yaml with consensus. - #[arg(long)] consensus_path: Option, } @@ -769,15 +768,8 @@ async fn main() -> anyhow::Result<()> { let opt = Cli::parse(); let mut config = if let Some(config_path) = opt.config_path.clone() { - let secrets_path = opt - .secrets_path - .clone() - .context("File based config can't be used partially please provide all paths")?; - let external_node_config_path = opt - .external_node_config_path - .clone() - .context("File based config can't be used partially please provide all paths")?; - + let secrets_path = opt.secrets_path.clone().unwrap(); + let external_node_config_path = opt.external_node_config_path.clone().unwrap(); ExternalNodeConfig::from_files( config_path, external_node_config_path, From 9ee546b064792e8eaca6502856ede8d53acfccee Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 15:56:26 +0200 Subject: [PATCH 18/56] Add commitment generator component Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 16 +++++++++---- .../src/configs/commitment_generator.rs | 8 +++++++ core/lib/config/src/configs/en_config.rs | 5 +--- core/lib/config/src/configs/experimental.rs | 2 +- core/lib/config/src/configs/general.rs | 7 +++--- core/lib/config/src/configs/mod.rs | 2 ++ .../src/commitment_generator.rs | 24 +++++++++++++++++++ core/lib/protobuf_config/src/en.rs | 10 +------- core/lib/protobuf_config/src/general.rs | 5 +++- core/lib/protobuf_config/src/lib.rs | 1 + .../proto/config/commitment_generator.proto | 7 ++++++ .../protobuf_config/src/proto/config/en.proto | 2 -- .../src/proto/config/general.proto | 2 ++ .../src/temp_config_store/mod.rs | 1 + 14 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 core/lib/config/src/configs/commitment_generator.rs create mode 100644 core/lib/protobuf_config/src/commitment_generator.rs create mode 100644 core/lib/protobuf_config/src/proto/config/commitment_generator.proto diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 62f0b34e5a58..76df29afc95a 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1089,8 +1089,10 @@ impl ExperimentalENConfig { tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), - commitment_generator_max_parallelism: external_node_config - .commitment_generator_max_parallelism, + commitment_generator_max_parallelism: general_config + .commitment_generator + .as_ref() + .map(|a| a.max_parallelism), }) } } @@ -1142,9 +1144,13 @@ pub struct ApiComponentConfig { } impl ApiComponentConfig { - fn from_configs(en_config: &ENConfig) -> Self { + fn from_configs(general_config: &GeneralConfig) -> Self { ApiComponentConfig { - tree_api_remote_url: en_config.tree_api_remote_url.clone(), + tree_api_remote_url: general_config + .api_config + .as_ref() + .map(|a| a.web3_json_rpc.tree_api_url.clone()) + .flatten(), } } } @@ -1244,7 +1250,7 @@ impl ExternalNodeConfig<()> { let experimental = ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; - let api_component = ApiComponentConfig::from_configs(&external_node_config); + let api_component = ApiComponentConfig::from_configs(&general_config); let tree_component = TreeComponentConfig::from_configs(&general_config); Ok(Self { diff --git a/core/lib/config/src/configs/commitment_generator.rs b/core/lib/config/src/configs/commitment_generator.rs new file mode 100644 index 000000000000..4a06bfdba63c --- /dev/null +++ b/core/lib/config/src/configs/commitment_generator.rs @@ -0,0 +1,8 @@ +use std::num::NonZeroU32; + +use serde::Deserialize; + +#[derive(Debug, Deserialize, Clone, PartialEq)] +pub struct CommitmentGeneratorConfig { + pub max_parallelism: NonZeroU32, +} diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index ab8e49617e84..5f003650fd41 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,4 +1,4 @@ -use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; +use std::num::{NonZeroU64, NonZeroUsize}; use serde::Deserialize; use zksync_basic_types::{ @@ -32,9 +32,6 @@ pub struct ENConfig { pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, - // En specific components - pub commitment_generator_max_parallelism: Option, - pub tree_api_remote_url: Option, pub snapshot_recovery: Option, pub pruning: Option, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 31bdf3a4730f..19e08dc2bb8a 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -36,7 +36,7 @@ impl Default for ExperimentalDBConfig { Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, reads_persistence_enabled: Self::default_protective_reads_persistence_enabled(), - processing_delay_ms: 0, + processing_delay_ms: Self::default_merkle_tree_processing_delay_ms(), include_indices_and_filters_in_block_cache: false, } } diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index ef02f557bc18..d5c8490ab2be 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -4,9 +4,9 @@ use crate::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, vm_runner::ProtectiveReadsWriterConfig, - FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig, - PrometheusConfig, ProofDataHandlerConfig, + CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, PostgresConfig, SnapshotsCreatorConfig, }; @@ -34,4 +34,5 @@ pub struct GeneralConfig { pub snapshot_creator: Option, pub observability: Option, pub protective_reads_writer_config: Option, + pub commitment_generator: Option, } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 990b0c858b2c..ae988abca2ce 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -1,6 +1,7 @@ // Public re-exports pub use self::{ api::ApiConfig, + commitment_generator::CommitmentGeneratorConfig, contract_verifier::ContractVerifierConfig, contracts::{ContractsConfig, EcosystemContracts}, database::{DBConfig, PostgresConfig}, @@ -25,6 +26,7 @@ pub use self::{ pub mod api; pub mod chain; +mod commitment_generator; pub mod consensus; pub mod contract_verifier; pub mod contracts; diff --git a/core/lib/protobuf_config/src/commitment_generator.rs b/core/lib/protobuf_config/src/commitment_generator.rs new file mode 100644 index 000000000000..23af3ccce76e --- /dev/null +++ b/core/lib/protobuf_config/src/commitment_generator.rs @@ -0,0 +1,24 @@ +use std::num::NonZeroU32; + +use anyhow::Context as _; +use zksync_config::configs::CommitmentGeneratorConfig; +use zksync_protobuf::{repr::ProtoRepr, required}; + +use crate::proto::commitment_generator as proto; + +impl ProtoRepr for proto::CommitmentGenerator { + type Type = CommitmentGeneratorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + max_parallelism: NonZeroU32::new( + *required(&self.max_parallelism).context("max_parallelism")?, + ) + .context("cannot be 0")?, + }) + } + fn build(this: &Self::Type) -> Self { + Self { + max_parallelism: Some(this.max_parallelism.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 8e3b99d1aaa8..4dbcae4fdbdd 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,5 +1,5 @@ use std::{ - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU64, NonZeroUsize}, str::FromStr, }; @@ -74,10 +74,6 @@ impl ProtoRepr for proto::ExternalNode { .and_then(|x| Ok(crate::proto::genesis::L1BatchCommitDataGeneratorMode::try_from(*x)?)) .context("l1_batch_commit_data_generator_mode")? .parse(), - commitment_generator_max_parallelism: self - .commitment_generator_max_parallelism - .and_then(NonZeroU32::new), - tree_api_remote_url: self.tree_api_remote_url.clone(), main_node_rate_limit_rps: self .main_node_rate_limit_rps .and_then(|a| NonZeroUsize::new(a as usize)), @@ -98,10 +94,6 @@ impl ProtoRepr for proto::ExternalNode { ) .into(), ), - tree_api_remote_url: this.tree_api_remote_url.clone(), - commitment_generator_max_parallelism: this - .commitment_generator_max_parallelism - .map(|a| a.get()), main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), pruning: this.pruning.as_ref().map(ProtoRepr::build), diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index ba2076a09a14..3583d77e5ce3 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -38,7 +38,9 @@ impl ProtoRepr for proto::GeneralConfig { .context("snapshot_creator")?, observability: read_optional_repr(&self.observability).context("observability")?, protective_reads_writer_config: read_optional_repr(&self.protective_reads_writer) - .context("vm_runner")?, + .context("protective_reads_writer_config")?, + commitment_generator: read_optional_repr(&self.commitment_generator) + .context("commitment_generator")?, }) } @@ -74,6 +76,7 @@ impl ProtoRepr for proto::GeneralConfig { .protective_reads_writer_config .as_ref() .map(ProtoRepr::build), + commitment_generator: this.commitment_generator.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index ebc0ae6b85a2..8c95f4cedf98 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -7,6 +7,7 @@ mod api; mod chain; mod circuit_breaker; +mod commitment_generator; mod consensus; mod contract_verifier; mod contracts; diff --git a/core/lib/protobuf_config/src/proto/config/commitment_generator.proto b/core/lib/protobuf_config/src/proto/config/commitment_generator.proto new file mode 100644 index 000000000000..62b9566e1866 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/commitment_generator.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package zksync.config.commitment_generator; + +message CommitmentGenerator { + optional uint32 max_parallelism = 1; +} diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 375872d36df4..7b64f75f270d 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -20,8 +20,6 @@ message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required - optional uint32 commitment_generator_max_parallelism = 4; // optional - optional string tree_api_remote_url = 5; // optional optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup optional SnapshotRecovery snapshot_recovery = 8; diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index b606417d129a..7dcedefd6a4d 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -14,6 +14,7 @@ import "zksync/config/observability.proto"; import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; import "zksync/config/vm_runner.proto"; +import "zksync/config/commitment_generator.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -37,4 +38,5 @@ message GeneralConfig { optional config.snapshot_creator.SnapshotsCreator snapshot_creator = 31; optional config.observability.Observability observability = 32; optional config.vm_runner.ProtectiveReadsWriter protective_reads_writer = 33; + optional config.commitment_generator.CommitmentGenerator commitment_generator = 34; } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index c1c8af29d964..fc9b165d782f 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -97,6 +97,7 @@ impl TempConfigStore { snapshot_creator: self.snapshot_creator.clone(), observability: self.observability.clone(), protective_reads_writer_config: self.protective_reads_writer_config.clone(), + commitment_generator: None, } } From e2fe4814e991bfdbfa473c939f82070d2729a2e8 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 17:46:21 +0200 Subject: [PATCH 19/56] Move snapshot recovery and prunning to the general config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 25 ++++---- core/bin/zksync_server/src/main.rs | 3 + core/lib/config/src/configs/en_config.rs | 20 +------ core/lib/config/src/configs/general.rs | 4 ++ core/lib/config/src/configs/mod.rs | 4 ++ core/lib/config/src/configs/pruning.rs | 11 ++++ .../config/src/configs/snapshot_recovery.rs | 10 ++++ core/lib/protobuf_config/src/en.rs | 58 +------------------ core/lib/protobuf_config/src/general.rs | 5 ++ core/lib/protobuf_config/src/lib.rs | 3 + .../protobuf_config/src/proto/config/en.proto | 15 ----- .../src/proto/config/general.proto | 4 ++ .../src/proto/config/pruning.proto | 10 ++++ .../src/proto/config/snapshot_recovery.proto | 10 ++++ core/lib/protobuf_config/src/pruning.rs | 28 +++++++++ .../protobuf_config/src/snapshot_recovery.rs | 28 +++++++++ .../src/temp_config_store/mod.rs | 14 +++-- 17 files changed, 144 insertions(+), 108 deletions(-) create mode 100644 core/lib/config/src/configs/pruning.rs create mode 100644 core/lib/config/src/configs/snapshot_recovery.rs create mode 100644 core/lib/protobuf_config/src/proto/config/pruning.proto create mode 100644 core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto create mode 100644 core/lib/protobuf_config/src/pruning.rs create mode 100644 core/lib/protobuf_config/src/snapshot_recovery.rs diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 76df29afc95a..4c46265a79a5 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -607,33 +607,33 @@ impl OptionalENConfig { default_l2_block_seal_queue_capacity ), l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, - snapshots_recovery_enabled: enconfig + snapshots_recovery_enabled: general_config .snapshot_recovery .as_ref() .map(|a| a.enabled) .unwrap_or_default(), snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( - enconfig.snapshot_recovery, + general_config.snapshot_recovery, postgres_max_concurrency, default_snapshots_recovery_postgres_max_concurrency ), - pruning_enabled: enconfig + pruning_enabled: general_config .pruning .as_ref() .map(|a| a.enabled) .unwrap_or_default(), pruning_chunk_size: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, chunk_size, default_pruning_chunk_size ), pruning_removal_delay_sec: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, removal_delay_sec, default_pruning_removal_delay_sec ), pruning_data_retention_sec: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, data_retention_sec, default_pruning_data_retention_sec ), @@ -1069,10 +1069,7 @@ impl ExperimentalENConfig { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } - pub fn from_configs( - general_config: &GeneralConfig, - external_node_config: &ENConfig, - ) -> anyhow::Result { + pub fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { Ok(Self { state_keeper_db_block_cache_capacity_mb: load_config_or_default!( general_config.db_config, @@ -1085,7 +1082,7 @@ impl ExperimentalENConfig { experimental.state_keeper_db_max_open_files ), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( - external_node_config.snapshot_recovery, + general_config.snapshot_recovery, tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), @@ -1149,8 +1146,7 @@ impl ApiComponentConfig { tree_api_remote_url: general_config .api_config .as_ref() - .map(|a| a.web3_json_rpc.tree_api_url.clone()) - .flatten(), + .and_then(|a| a.web3_json_rpc.tree_api_url.clone()), } } } @@ -1247,8 +1243,7 @@ impl ExternalNodeConfig<()> { .max_connections()?, }; let observability = ObservabilityENConfig::from_configs(&general_config)?; - let experimental = - ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; + let experimental = ExperimentalENConfig::from_configs(&general_config)?; let api_component = ApiComponentConfig::from_configs(&general_config); let tree_component = TreeComponentConfig::from_configs(&general_config); diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index f1eedd592386..0c6e05c2059a 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -308,5 +308,8 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + commitment_generator: None, + pruning: None, + snapshot_recovery: None, }) } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 5f003650fd41..32dc5b7c7b49 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,25 +1,10 @@ -use std::num::{NonZeroU64, NonZeroUsize}; +use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::{ commitment::L1BatchCommitmentMode, url::SensitiveUrl, L1ChainId, L2ChainId, }; -#[derive(Debug, Clone, PartialEq, Deserialize)] -pub struct Pruning { - pub enabled: bool, - pub chunk_size: Option, - pub removal_delay_sec: Option, - pub data_retention_sec: Option, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -pub struct SnapshotRecovery { - pub enabled: bool, - pub postgres_max_concurrency: Option, - pub tree_chunk_size: Option, -} - /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { @@ -31,7 +16,4 @@ pub struct ENConfig { // Main node configuration pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, - - pub snapshot_recovery: Option, - pub pruning: Option, } diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index d5c8490ab2be..879ff31d5046 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -3,6 +3,8 @@ use crate::{ chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig}, fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, + pruning::PruningConfig, + snapshot_recovery::SnapshotRecoveryConfig, vm_runner::ProtectiveReadsWriterConfig, CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, @@ -35,4 +37,6 @@ pub struct GeneralConfig { pub observability: Option, pub protective_reads_writer_config: Option, pub commitment_generator: Option, + pub snapshot_recovery: Option, + pub pruning: Option, } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index ae988abca2ce..9e04f483357f 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -18,7 +18,9 @@ pub use self::{ object_store::ObjectStoreConfig, observability::{ObservabilityConfig, OpentelemetryConfig}, proof_data_handler::ProofDataHandlerConfig, + pruning::PruningConfig, secrets::{DatabaseSecrets, L1Secrets, Secrets}, + snapshot_recovery::SnapshotRecoveryConfig, snapshots_creator::SnapshotsCreatorConfig, utils::PrometheusConfig, vm_runner::ProtectiveReadsWriterConfig, @@ -47,7 +49,9 @@ pub mod house_keeper; pub mod object_store; pub mod observability; pub mod proof_data_handler; +pub mod pruning; pub mod secrets; +pub mod snapshot_recovery; pub mod snapshots_creator; pub mod utils; pub mod vm_runner; diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs new file mode 100644 index 000000000000..ae09d43b3bfc --- /dev/null +++ b/core/lib/config/src/configs/pruning.rs @@ -0,0 +1,11 @@ +use std::num::NonZeroU64; + +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct PruningConfig { + pub enabled: bool, + pub chunk_size: Option, + pub removal_delay_sec: Option, + pub data_retention_sec: Option, +} diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs new file mode 100644 index 000000000000..96389eb2fd2e --- /dev/null +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -0,0 +1,10 @@ +use std::num::NonZeroUsize; + +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct SnapshotRecoveryConfig { + pub enabled: bool, + pub postgres_max_concurrency: Option, + pub tree_chunk_size: Option, +} diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 4dbcae4fdbdd..b72a5b142cfb 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,58 +1,11 @@ -use std::{ - num::{NonZeroU64, NonZeroUsize}, - str::FromStr, -}; +use std::{num::NonZeroUsize, str::FromStr}; use anyhow::Context; use zksync_basic_types::{url::SensitiveUrl, L1ChainId, L2ChainId}; -use zksync_config::configs::en_config::{ENConfig, Pruning, SnapshotRecovery}; +use zksync_config::configs::en_config::ENConfig; use zksync_protobuf::{required, ProtoRepr}; -use crate::{proto::en as proto, read_optional_repr}; - -impl ProtoRepr for proto::Pruning { - type Type = Pruning; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - chunk_size: self.chunk_size, - removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), - data_retention_sec: self.data_retention_sec, - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - enabled: Some(this.enabled), - chunk_size: this.chunk_size, - removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), - data_retention_sec: this.data_retention_sec, - } - } -} - -impl ProtoRepr for proto::SnapshotRecovery { - type Type = SnapshotRecovery; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - postgres_max_concurrency: self - .postgres_max_concurrency - .and_then(|a| NonZeroUsize::new(a as usize)), - tree_chunk_size: self.tree_chunk_size, - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - enabled: Some(this.enabled), - postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), - tree_chunk_size: this.tree_chunk_size, - } - } -} +use crate::proto::en as proto; impl ProtoRepr for proto::ExternalNode { type Type = ENConfig; @@ -77,9 +30,6 @@ impl ProtoRepr for proto::ExternalNode { main_node_rate_limit_rps: self .main_node_rate_limit_rps .and_then(|a| NonZeroUsize::new(a as usize)), - pruning: read_optional_repr(&self.pruning).context("pruning")?, - snapshot_recovery: read_optional_repr(&self.snapshot_recovery) - .context("snapshot_recovery")?, }) } @@ -95,8 +45,6 @@ impl ProtoRepr for proto::ExternalNode { .into(), ), main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), - snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), - pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index 3583d77e5ce3..1cb62b83f155 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -41,6 +41,9 @@ impl ProtoRepr for proto::GeneralConfig { .context("protective_reads_writer_config")?, commitment_generator: read_optional_repr(&self.commitment_generator) .context("commitment_generator")?, + pruning: read_optional_repr(&self.pruning).context("pruning")?, + snapshot_recovery: read_optional_repr(&self.snapshot_recovery) + .context("snapshot_recovery")?, }) } @@ -77,6 +80,8 @@ impl ProtoRepr for proto::GeneralConfig { .as_ref() .map(ProtoRepr::build), commitment_generator: this.commitment_generator.as_ref().map(ProtoRepr::build), + snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), + pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index 8c95f4cedf98..14e4f5455f5f 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -23,8 +23,11 @@ mod observability; mod proof_data_handler; pub mod proto; mod prover; +mod pruning; mod secrets; mod snapshots_creator; + +mod snapshot_recovery; pub mod testonly; #[cfg(test)] mod tests; diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 7b64f75f270d..ac7cb59b156a 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -3,25 +3,10 @@ import "zksync/config/genesis.proto"; package zksync.config.en; -message SnapshotRecovery { - optional bool enabled = 1; - optional uint64 postgres_max_concurrency = 2; - optional uint64 tree_chunk_size = 3; -} - -message Pruning { - optional bool enabled = 1; - optional uint32 chunk_size = 2; - optional uint64 removal_delay_sec = 3; - optional uint64 data_retention_sec = 4; -} - message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup - optional SnapshotRecovery snapshot_recovery = 8; - optional Pruning pruning = 9; } diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index 7dcedefd6a4d..5465a441ad54 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -15,6 +15,8 @@ import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; import "zksync/config/vm_runner.proto"; import "zksync/config/commitment_generator.proto"; +import "zksync/config/snapshot_recovery.proto"; +import "zksync/config/pruning.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -39,4 +41,6 @@ message GeneralConfig { optional config.observability.Observability observability = 32; optional config.vm_runner.ProtectiveReadsWriter protective_reads_writer = 33; optional config.commitment_generator.CommitmentGenerator commitment_generator = 34; + optional config.snapshot_recovery.SnapshotRecovery snapshot_recovery = 35; + optional config.pruning.Pruning pruning = 36; } diff --git a/core/lib/protobuf_config/src/proto/config/pruning.proto b/core/lib/protobuf_config/src/proto/config/pruning.proto new file mode 100644 index 000000000000..351f353bf060 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/pruning.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.pruning; + +message Pruning { + optional bool enabled = 1; + optional uint32 chunk_size = 2; + optional uint64 removal_delay_sec = 3; + optional uint64 data_retention_sec = 4; +} diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto new file mode 100644 index 000000000000..da4b4b585bff --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.snapshot_recovery; + +message SnapshotRecovery { + optional bool enabled = 1; + optional uint64 postgres_max_concurrency = 2; + optional uint64 tree_chunk_size = 3; +} + diff --git a/core/lib/protobuf_config/src/pruning.rs b/core/lib/protobuf_config/src/pruning.rs new file mode 100644 index 000000000000..ed0ebb10b92f --- /dev/null +++ b/core/lib/protobuf_config/src/pruning.rs @@ -0,0 +1,28 @@ +use std::num::NonZeroU64; + +use zksync_config::configs::PruningConfig; +use zksync_protobuf::ProtoRepr; + +use crate::proto::pruning as proto; + +impl ProtoRepr for proto::Pruning { + type Type = PruningConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + chunk_size: self.chunk_size, + removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), + data_retention_sec: self.data_retention_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + chunk_size: this.chunk_size, + removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), + data_retention_sec: this.data_retention_sec, + } + } +} diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs new file mode 100644 index 000000000000..19e2862d4e99 --- /dev/null +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -0,0 +1,28 @@ +use std::num::NonZeroUsize; + +use zksync_config::configs::SnapshotRecoveryConfig; +use zksync_protobuf::ProtoRepr; + +use crate::proto::snapshot_recovery as proto; + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecoveryConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + postgres_max_concurrency: self + .postgres_max_concurrency + .and_then(|a| NonZeroUsize::new(a as usize)), + tree_chunk_size: self.tree_chunk_size, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), + tree_chunk_size: this.tree_chunk_size, + } + } +} diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index fc9b165d782f..6476172c3767 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -11,9 +11,10 @@ use zksync_config::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, wallets::{AddressWallet, EthSender, StateKeeper, Wallet, Wallets}, - FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, GeneralConfig, - ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, + CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + GeneralConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, + ProtectiveReadsWriterConfig, PruningConfig, SnapshotRecoveryConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -71,6 +72,9 @@ pub struct TempConfigStore { pub observability: Option, pub snapshot_creator: Option, pub protective_reads_writer_config: Option, + pub commitment_generator: Option, + pub pruning: Option, + pub snapshot_recovery: Option, } impl TempConfigStore { @@ -97,7 +101,9 @@ impl TempConfigStore { snapshot_creator: self.snapshot_creator.clone(), observability: self.observability.clone(), protective_reads_writer_config: self.protective_reads_writer_config.clone(), - commitment_generator: None, + commitment_generator: self.commitment_generator.clone(), + snapshot_recovery: self.snapshot_recovery.clone(), + pruning: self.pruning.clone(), } } From 71dff3237272d60eb6f3ddbb17dc38b083c2279e Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 18:13:21 +0200 Subject: [PATCH 20/56] Fix file based config Signed-off-by: Danil --- etc/env/file_based/external_node.yaml | 10 ---------- etc/env/file_based/general.yaml | 13 +++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index edb2b4af597f..675baf739686 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,16 +1,6 @@ l1_chain_id: 9 l2_chain_id: 270 l1_batch_commit_data_generator_mode: Rollup -commitment_generator_max_parallelism: 10 main_node_url: http://localhost:3050 main_node_rate_limit_rps: 1000 -snapshot_recovery: - enabled: true - postgres_max_concurrency: 10 - tree_chunk_size: 200000 -pruning: - enabled: true - chunk_size: 10 - removal_delay_sec: 60 - data_retention_sec: 3600 diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index f1ed009f01c7..5766af85c683 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -325,3 +325,16 @@ observability: protective_reads_writer: protective_reads_db_path: "./db/main/protective_reads" protective_reads_window_size: 3 + +snapshot_recovery: + enabled: true + postgres_max_concurrency: 10 + tree_chunk_size: 200000 +pruning: + enabled: true + chunk_size: 10 + removal_delay_sec: 60 + data_retention_sec: 3600 + +commitment_generator: + max_parallelism: 10 From b33173c4f8b3973fc5c3b5a88b9d5fc28ee5cf76 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 6 Jun 2024 15:09:38 +0400 Subject: [PATCH 21/56] Support for api namespaces Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 5 +---- core/bin/zksync_server/src/node_builder.rs | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 4c46265a79a5..a574a01ad291 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -444,10 +444,7 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| { - let result: Result, _> = a.iter().map(|a| serde_json::from_str(a)).collect(); - result - }) + .map(|a: Vec| serde_json::from_str(&a.join(","))) .transpose()?; Ok(OptionalENConfig { diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 1dc9e1de554e..548445b02a4e 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -295,9 +295,7 @@ impl MainNodeBuilder { let with_debug_namespace = state_keeper_config.save_call_traces; let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { - let result: Result, _> = - namespaces.iter().map(|a| serde_json::from_str(a)).collect(); - result? + serde_json::from_str(&namespaces.join(","))? } else { Namespace::DEFAULT.to_vec() }; From 0583b69ef4d820edd1ac7e4d96acd7cd07e608ef Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 6 Jun 2024 16:47:32 +0400 Subject: [PATCH 22/56] Fix general.yaml Signed-off-by: Danil --- etc/env/file_based/general.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 61c67586b1a7..3ceda661cff5 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -63,6 +63,7 @@ api: estimate_gas_scale_factor: 1.2 estimate_gas_acceptable_overestimation: 1000 max_tx_size: 1000000 + api_namespaces: [ eth,net,web3,zks,pubsub ] max_response_body_size_overrides: - method: eth_getTransactionReceipt # no size specified, meaning no size limit - method: zks_getProof From 0d39b60693482dac486cc970684be0b435a16db5 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 10:53:27 +0400 Subject: [PATCH 23/56] Update snapshot recovery Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 5 +++++ core/lib/config/src/configs/snapshot_recovery.rs | 3 +++ .../src/proto/config/snapshot_recovery.proto | 2 ++ core/lib/protobuf_config/src/snapshot_recovery.rs | 10 ++++++++++ 4 files changed, 20 insertions(+) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index d1ed4dea1ec9..e61dac88bee8 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1088,11 +1088,16 @@ impl ExperimentalENConfig { general_config.db_config, experimental.state_keeper_db_max_open_files ), + snapshots_recovery_l1_batch: load_config!(general_config.snapshot_recovery, l1_batch), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( general_config.snapshot_recovery, tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), + snapshots_recovery_tree_parallel_persistence_buffer: load_config!( + general_config.snapshot_recovery, + tree_parallel_persistence_buffer + ), commitment_generator_max_parallelism: general_config .commitment_generator .as_ref() diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index 96389eb2fd2e..1ac5d6a559c9 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -1,10 +1,13 @@ use std::num::NonZeroUsize; use serde::Deserialize; +use zksync_basic_types::L1BatchNumber; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct SnapshotRecoveryConfig { pub enabled: bool, pub postgres_max_concurrency: Option, pub tree_chunk_size: Option, + pub l1_batch: Option, + pub tree_parallel_persistence_buffer: Option, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index da4b4b585bff..f74282608d1a 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -6,5 +6,7 @@ message SnapshotRecovery { optional bool enabled = 1; optional uint64 postgres_max_concurrency = 2; optional uint64 tree_chunk_size = 3; + optional uint32 l1_batch = 4; + optional uint64 tree_parallel_persistence_buffer = 5; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 19e2862d4e99..269261ed54a2 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -1,5 +1,6 @@ use std::num::NonZeroUsize; +use zksync_basic_types::L1BatchNumber; use zksync_config::configs::SnapshotRecoveryConfig; use zksync_protobuf::ProtoRepr; @@ -15,6 +16,11 @@ impl ProtoRepr for proto::SnapshotRecovery { .postgres_max_concurrency .and_then(|a| NonZeroUsize::new(a as usize)), tree_chunk_size: self.tree_chunk_size, + tree_parallel_persistence_buffer: self + .tree_parallel_persistence_buffer + .and_then(|a| NonZeroUsize::new(a as usize)), + + l1_batch: self.l1_batch.map(L1BatchNumber), }) } @@ -23,6 +29,10 @@ impl ProtoRepr for proto::SnapshotRecovery { enabled: Some(this.enabled), postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), tree_chunk_size: this.tree_chunk_size, + l1_batch: this.l1_batch.map(|a| a.0), + tree_parallel_persistence_buffer: this + .tree_parallel_persistence_buffer + .map(|a| a.get() as u64), } } } From af21794836cf8cdfed9b32cf71c4130475eb8347 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 14:26:43 +0400 Subject: [PATCH 24/56] Split snapshot recovery to smaller files Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 6 +- core/lib/config/src/configs/experimental.rs | 2 + core/lib/config/src/configs/pruning.rs | 9 +++ .../config/src/configs/snapshot_recovery.rs | 34 +++++++- .../src/proto/config/snapshot_recovery.proto | 14 +++- .../protobuf_config/src/snapshot_recovery.rs | 77 +++++++++++++++---- .../src/temp_config_store/mod.rs | 2 +- 7 files changed, 119 insertions(+), 25 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e61dac88bee8..10e32c17ca69 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -611,7 +611,7 @@ impl OptionalENConfig { .unwrap_or_default(), snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( general_config.snapshot_recovery, - postgres_max_concurrency, + postgres.max_concurrency, default_snapshots_recovery_postgres_max_concurrency ), pruning_enabled: general_config @@ -1091,12 +1091,12 @@ impl ExperimentalENConfig { snapshots_recovery_l1_batch: load_config!(general_config.snapshot_recovery, l1_batch), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( general_config.snapshot_recovery, - tree_chunk_size, + tree.chunk_size, default_snapshots_recovery_tree_chunk_size ), snapshots_recovery_tree_parallel_persistence_buffer: load_config!( general_config.snapshot_recovery, - tree_parallel_persistence_buffer + tree.parallel_persistence_buffer ), commitment_generator_max_parallelism: general_config .commitment_generator diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 19e08dc2bb8a..46d93cdf18f6 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -50,9 +50,11 @@ impl ExperimentalDBConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * super::BYTES_IN_MEGABYTE } + const fn default_protective_reads_persistence_enabled() -> bool { true } + const fn default_merkle_tree_processing_delay_ms() -> u64 { 100 } diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs index ae09d43b3bfc..f1b3a8e74558 100644 --- a/core/lib/config/src/configs/pruning.rs +++ b/core/lib/config/src/configs/pruning.rs @@ -5,7 +5,16 @@ use serde::Deserialize; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct PruningConfig { pub enabled: bool, + /// Chunk size for multi-get operations. Can speed up loading data for the Merkle tree on some environments, + /// but the effects vary wildly depending on the setup (e.g., the filesystem used). pub chunk_size: Option, + /// Delta between soft- and hard-removing data from Postgres. Should be reasonably large (order of 60 seconds). + /// The default value is 60 seconds. pub removal_delay_sec: Option, + /// If set, L1 batches will be pruned after the batch timestamp is this old (in seconds). Note that an L1 batch + /// may be temporarily retained for other reasons; e.g., a batch cannot be pruned until it is executed on L1, + /// which happens roughly 24 hours after its generation on the mainnet. Thus, in practice this value can specify + /// the retention period greater than that implicitly imposed by other criteria (e.g., 7 or 30 days). + /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 1 hour. pub data_retention_sec: Option, } diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index 1ac5d6a559c9..d2cf0da190d2 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -3,11 +3,39 @@ use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::L1BatchNumber; +#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +pub struct Tree { + /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. + /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds). + /// + /// **Important.** This value cannot be changed in the middle of tree recovery (i.e., if a node is stopped in the middle + /// of recovery and then restarted with a different config). + pub chunk_size: Option, + /// Buffer capacity for parallel persistence operations. Should be reasonably small since larger buffer means more RAM usage; + /// buffer elements are persisted tree chunks. OTOH, small buffer can lead to persistence parallelization being inefficient. + /// + /// If not set, parallel persistence will be disabled. + pub parallel_persistence_buffer: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +pub struct Postgres { + /// Maximum concurrency factor for the concurrent parts of snapshot recovery for Postgres. It may be useful to + /// reduce this factor to about 5 if snapshot recovery overloads I/O capacity of the node. Conversely, + /// if I/O capacity of your infra is high, you may increase concurrency to speed up Postgres recovery. + pub max_concurrency: Option, +} + #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct SnapshotRecoveryConfig { + /// Enables application-level snapshot recovery. Required to start a node that was recovered from a snapshot, + /// or to initialize a node from a snapshot. Has no effect if a node that was initialized from a Postgres dump + /// or was synced from genesis. + /// + /// This is an experimental and incomplete feature; do not use unless you know what you're doing. pub enabled: bool, - pub postgres_max_concurrency: Option, - pub tree_chunk_size: Option, + /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing. pub l1_batch: Option, - pub tree_parallel_persistence_buffer: Option, + pub tree: Tree, + pub postgres: Postgres, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index f74282608d1a..5f217723f8a7 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -2,11 +2,19 @@ syntax = "proto3"; package zksync.config.snapshot_recovery; +message Tree { + optional uint64 chunk_size = 1; + optional uint64 parallel_persistence_buffer = 2; +} + +message Postgres { + optional uint64 max_concurrency = 1; +} + message SnapshotRecovery { optional bool enabled = 1; - optional uint64 postgres_max_concurrency = 2; - optional uint64 tree_chunk_size = 3; + optional Postgres postgres = 2; + optional Tree tree = 3; optional uint32 l1_batch = 4; - optional uint64 tree_parallel_persistence_buffer = 5; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 269261ed54a2..7a84d1a158c6 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -1,38 +1,85 @@ use std::num::NonZeroUsize; +use anyhow::Context; use zksync_basic_types::L1BatchNumber; -use zksync_config::configs::SnapshotRecoveryConfig; +use zksync_config::configs::{ + snapshot_recovery::{Postgres, Tree}, + SnapshotRecoveryConfig, +}; use zksync_protobuf::ProtoRepr; -use crate::proto::snapshot_recovery as proto; +use crate::{proto::snapshot_recovery as proto, read_optional_repr}; -impl ProtoRepr for proto::SnapshotRecovery { - type Type = SnapshotRecoveryConfig; +impl ProtoRepr for proto::Tree { + type Type = Tree; fn read(&self) -> anyhow::Result { Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - postgres_max_concurrency: self - .postgres_max_concurrency + chunk_size: self.chunk_size, + parallel_persistence_buffer: self + .parallel_persistence_buffer .and_then(|a| NonZeroUsize::new(a as usize)), - tree_chunk_size: self.tree_chunk_size, - tree_parallel_persistence_buffer: self - .tree_parallel_persistence_buffer + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + chunk_size: this.chunk_size, + parallel_persistence_buffer: this.parallel_persistence_buffer.map(|a| a.get() as u64), + } + } +} + +impl ProtoRepr for proto::Postgres { + type Type = Postgres; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + max_concurrency: self + .max_concurrency .and_then(|a| NonZeroUsize::new(a as usize)), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + max_concurrency: this.max_concurrency.map(|a| a.get() as u64), + } + } +} + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecoveryConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + tree: read_optional_repr(&self.tree) + .context("tree")? + .unwrap_or_default(), + postgres: read_optional_repr(&self.postgres) + .context("postgres")? + .unwrap_or_default(), l1_batch: self.l1_batch.map(L1BatchNumber), }) } fn build(this: &Self::Type) -> Self { + let tree = if this.tree == Tree::default() { + None + } else { + Some(this.tree.clone()) + }; + let postgres = if this.postgres == Postgres::default() { + None + } else { + Some(this.postgres.clone()) + }; Self { enabled: Some(this.enabled), - postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), - tree_chunk_size: this.tree_chunk_size, + postgres: postgres.as_ref().map(ProtoRepr::build), + tree: tree.as_ref().map(ProtoRepr::build), l1_batch: this.l1_batch.map(|a| a.0), - tree_parallel_persistence_buffer: this - .tree_parallel_persistence_buffer - .map(|a| a.get() as u64), } } } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index ce1fbf56cb6b..979f74a26d7e 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -34,7 +34,7 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { } pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { - let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; + let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; decode_yaml_repr::(&yaml) } From 3677fb8ef15c26d6284711a9846d10ed276daf9d Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 15:15:24 +0400 Subject: [PATCH 25/56] Fix tets Signed-off-by: Danil --- etc/env/file_based/general.yaml | 7 +++++-- prover/config/src/lib.rs | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 3ceda661cff5..5e2c138f5a54 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -330,8 +330,11 @@ protective_reads_writer: snapshot_recovery: enabled: true - postgres_max_concurrency: 10 - tree_chunk_size: 200000 + postgres: + max_concurrency: 10 + tree: + chunk_size: 200000 + parallel_persistence_buffer: 1 pruning: enabled: true chunk_size: 10 diff --git a/prover/config/src/lib.rs b/prover/config/src/lib.rs index 8614f1677bda..3a913ab6ef06 100644 --- a/prover/config/src/lib.rs +++ b/prover/config/src/lib.rs @@ -49,6 +49,9 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + commitment_generator: None, + pruning: None, + snapshot_recovery: None, }) } From a1bf8ed9d6a0712f7a1853651066d9f177e46adb Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 15:52:12 +0400 Subject: [PATCH 26/56] Fix observability Signed-off-by: Danil --- core/bin/external_node/src/config/observability.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index efec5a8a0d87..b66e471f0087 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -121,11 +121,11 @@ impl ObservabilityENConfig { (None, None, LogFormat::default(), None) }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = - if let Some(api) = general_config.api_config.as_ref() { + if let Some(prometheus) = general_config.prometheus_config.as_ref() { ( - Some(api.prometheus.listener_port), - Some(api.prometheus.pushgateway_url.clone()), - api.prometheus.push_interval_ms.unwrap_or_default(), + Some(prometheus.listener_port), + Some(prometheus.pushgateway_url.clone()), + prometheus.push_interval_ms.unwrap_or_default(), ) } else { (None, None, 0) From 9bb522fdfe74e489c2388ce3dd466e5452a15a82 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 18:32:00 +0400 Subject: [PATCH 27/56] Set more requires Signed-off-by: Danil --- core/bin/external_node/src/main.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 931cd2b271eb..82d0775a3eed 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -703,13 +703,17 @@ struct Cli { #[arg(long, default_value = "all")] components: ComponentsToRun, /// Path to the yaml config. If set, it will be used instead of env vars. - #[arg(long, requires = "secrets_path")] + #[arg( + long, + requires = "secrets_path", + requires = "external_node_config_path" + )] config_path: Option, /// Path to the yaml with secrets. If set, it will be used instead of env vars. - #[arg(long, requires = "external_node_config_path")] + #[arg(long, requires = "config_path", requires = "external_node_config_path")] secrets_path: Option, /// Path to the yaml with genesis. If set, it will be used instead of env vars. - #[arg(long, requires = "config_path")] + #[arg(long, requires = "config_path", requires = "secrets_path")] external_node_config_path: Option, /// Path to the yaml with consensus. consensus_path: Option, From 4d38eacd77c879514a291baf076c6c79bde31f00 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 11:46:50 +0200 Subject: [PATCH 28/56] Some nit fixes Signed-off-by: Danil --- core/lib/config/src/configs/pruning.rs | 3 +-- core/lib/config/src/configs/snapshot_recovery.rs | 8 ++++---- .../src/proto/config/snapshot_recovery.proto | 1 - core/lib/protobuf_config/src/snapshot_recovery.rs | 10 +++++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs index f1b3a8e74558..d2a5b0e5e9df 100644 --- a/core/lib/config/src/configs/pruning.rs +++ b/core/lib/config/src/configs/pruning.rs @@ -5,8 +5,7 @@ use serde::Deserialize; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct PruningConfig { pub enabled: bool, - /// Chunk size for multi-get operations. Can speed up loading data for the Merkle tree on some environments, - /// but the effects vary wildly depending on the setup (e.g., the filesystem used). + /// Number of L1 batches pruned at a time. pub chunk_size: Option, /// Delta between soft- and hard-removing data from Postgres. Should be reasonably large (order of 60 seconds). /// The default value is 60 seconds. diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index d2cf0da190d2..ef2ba49011dd 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use zksync_basic_types::L1BatchNumber; #[derive(Debug, Clone, PartialEq, Deserialize, Default)] -pub struct Tree { +pub struct TreeRecoveryConfig { /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds). /// @@ -19,7 +19,7 @@ pub struct Tree { } #[derive(Debug, Clone, PartialEq, Deserialize, Default)] -pub struct Postgres { +pub struct PostgresRecoveryConfig { /// Maximum concurrency factor for the concurrent parts of snapshot recovery for Postgres. It may be useful to /// reduce this factor to about 5 if snapshot recovery overloads I/O capacity of the node. Conversely, /// if I/O capacity of your infra is high, you may increase concurrency to speed up Postgres recovery. @@ -36,6 +36,6 @@ pub struct SnapshotRecoveryConfig { pub enabled: bool, /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing. pub l1_batch: Option, - pub tree: Tree, - pub postgres: Postgres, + pub tree: TreeRecoveryConfig, + pub postgres: PostgresRecoveryConfig, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index 5f217723f8a7..5faf0399ba69 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -17,4 +17,3 @@ message SnapshotRecovery { optional Tree tree = 3; optional uint32 l1_batch = 4; } - diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 7a84d1a158c6..237f6057d794 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -3,7 +3,7 @@ use std::num::NonZeroUsize; use anyhow::Context; use zksync_basic_types::L1BatchNumber; use zksync_config::configs::{ - snapshot_recovery::{Postgres, Tree}, + snapshot_recovery::{PostgresRecoveryConfig, TreeRecoveryConfig}, SnapshotRecoveryConfig, }; use zksync_protobuf::ProtoRepr; @@ -11,7 +11,7 @@ use zksync_protobuf::ProtoRepr; use crate::{proto::snapshot_recovery as proto, read_optional_repr}; impl ProtoRepr for proto::Tree { - type Type = Tree; + type Type = TreeRecoveryConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { @@ -31,7 +31,7 @@ impl ProtoRepr for proto::Tree { } impl ProtoRepr for proto::Postgres { - type Type = Postgres; + type Type = PostgresRecoveryConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { @@ -65,12 +65,12 @@ impl ProtoRepr for proto::SnapshotRecovery { } fn build(this: &Self::Type) -> Self { - let tree = if this.tree == Tree::default() { + let tree = if this.tree == TreeRecoveryConfig::default() { None } else { Some(this.tree.clone()) }; - let postgres = if this.postgres == Postgres::default() { + let postgres = if this.postgres == PostgresRecoveryConfig::default() { None } else { Some(this.postgres.clone()) From c99be45334f07febb34bb968d1677fe2841341f7 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 21:31:59 +0200 Subject: [PATCH 29/56] Switch from serde to strum for namespaces Signed-off-by: Danil --- Cargo.lock | 2 ++ Cargo.toml | 1 + core/bin/external_node/src/config/mod.rs | 2 +- core/bin/zksync_server/src/node_builder.rs | 5 ++++- core/node/api_server/Cargo.toml | 2 ++ core/node/api_server/src/web3/mod.rs | 3 ++- prover/Cargo.lock | 2 ++ 7 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a99150fe01c0..f123180cdf50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8807,6 +8807,8 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "strum", + "strum_macros", "test-casing", "thiserror", "thread_local", diff --git a/Cargo.toml b/Cargo.toml index 5d9f6adf37ad..216030b56a2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,6 +159,7 @@ sqlx = "0.7.3" static_assertions = "1.1" structopt = "0.3.20" strum = "0.24" +strum_macros = "0.24" tempfile = "3.0.2" test-casing = "0.1.2" test-log = "0.2.15" diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 10e32c17ca69..e83d3d679b29 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -444,7 +444,7 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| serde_json::from_str(&a.join(","))) + .map(|a: Vec| a.iter().map(|a| a.parse()).collect::>()) .transpose()?; Ok(OptionalENConfig { diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 374126176b52..0206f7c3a61f 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -293,7 +293,10 @@ impl MainNodeBuilder { let with_debug_namespace = state_keeper_config.save_call_traces; let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { - serde_json::from_str(&namespaces.join(","))? + namespaces + .iter() + .map(|a| a.parse()) + .collect::>()? } else { Namespace::DEFAULT.to_vec() }; diff --git a/core/node/api_server/Cargo.toml b/core/node/api_server/Cargo.toml index b826a8b40f21..6f0fad424a22 100644 --- a/core/node/api_server/Cargo.toml +++ b/core/node/api_server/Cargo.toml @@ -48,6 +48,8 @@ pin-project-lite.workspace = true hex.workspace = true http.workspace = true tower.workspace = true +strum.workspace = true +strum_macros.workspace = true tower-http = { workspace = true, features = ["cors", "metrics"] } lru.workspace = true diff --git a/core/node/api_server/src/web3/mod.rs b/core/node/api_server/src/web3/mod.rs index b86666ea6868..32ca1816254d 100644 --- a/core/node/api_server/src/web3/mod.rs +++ b/core/node/api_server/src/web3/mod.rs @@ -86,8 +86,9 @@ enum ApiTransport { Http(SocketAddr), } -#[derive(Debug, Deserialize, Clone, PartialEq)] +#[derive(Debug, Deserialize, Clone, PartialEq, strum_macros::EnumString)] #[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum Namespace { Eth, Net, diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 44c2a8b8395f..ee9933ade209 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -9014,6 +9014,8 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "strum", + "strum_macros", "thiserror", "thread_local", "tokio", From 9fb316435e1db498e2dffb8c7aea2d5d6843176d Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 19 Jun 2024 17:51:52 +0200 Subject: [PATCH 30/56] Update comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 4 ++-- core/lib/config/src/configs/commitment_generator.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e83d3d679b29..6ff22fda24f7 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -436,7 +436,7 @@ pub(crate) struct OptionalENConfig { /// may be temporarily retained for other reasons; e.g., a batch cannot be pruned until it is executed on L1, /// which happens roughly 24 hours after its generation on the mainnet. Thus, in practice this value can specify /// the retention period greater than that implicitly imposed by other criteria (e.g., 7 or 30 days). - /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 1 hour. + /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 7 days. #[serde(default = "OptionalENConfig::default_pruning_data_retention_sec")] pruning_data_retention_sec: u64, } @@ -794,7 +794,7 @@ impl OptionalENConfig { } fn default_pruning_data_retention_sec() -> u64 { - 3_600 // 1 hour + 3_600 * 24 * 7 // 7 days } fn from_env() -> anyhow::Result { diff --git a/core/lib/config/src/configs/commitment_generator.rs b/core/lib/config/src/configs/commitment_generator.rs index 4a06bfdba63c..9ec4d805b8fe 100644 --- a/core/lib/config/src/configs/commitment_generator.rs +++ b/core/lib/config/src/configs/commitment_generator.rs @@ -4,5 +4,7 @@ use serde::Deserialize; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct CommitmentGeneratorConfig { + /// Maximum degree of parallelism during commitment generation, i.e., the maximum number of L1 batches being processed in parallel. + /// If not specified, commitment generator will use a value roughly equal to the number of CPU cores with some clamping applied. pub max_parallelism: NonZeroU32, } From b04fc651221b82a1b3777695d5280fb546bb1fee Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 14:12:19 +0200 Subject: [PATCH 31/56] Update curve-dalek Signed-off-by: Danil --- Cargo.lock | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd76c5c65f5a..9480bd3a7a0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1509,16 +1509,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -4387,12 +4386,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "platforms" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" - [[package]] name = "plotters" version = "0.3.5" From 058322fc9e0cf66588ac68ea9feeedb141314bb8 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 12:35:15 +0200 Subject: [PATCH 32/56] Split to experimental and normal config for tree Signed-off-by: Danil --- .../src/proto/config/experimental.proto | 5 ++ .../src/proto/config/snapshot_recovery.proto | 5 +- .../protobuf_config/src/snapshot_recovery.rs | 63 +++++++++++-------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/core/lib/protobuf_config/src/proto/config/experimental.proto b/core/lib/protobuf_config/src/proto/config/experimental.proto index ddcb4265319d..6f9ec426d8bb 100644 --- a/core/lib/protobuf_config/src/proto/config/experimental.proto +++ b/core/lib/protobuf_config/src/proto/config/experimental.proto @@ -12,3 +12,8 @@ message DB { optional uint64 processing_delay_ms = 4; optional bool include_indices_and_filters_in_block_cache = 5; } + +// Experimental part of the Snapshot recovery configuration. +message SnapshotRecovery { + optional uint64 tree_recovery_parallel_persistence_buffer = 1; +} diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index 5faf0399ba69..9eceda12ad86 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -1,10 +1,11 @@ syntax = "proto3"; +import "zksync/config/object_store.proto"; +import "zksync/config/experimental.proto"; package zksync.config.snapshot_recovery; message Tree { optional uint64 chunk_size = 1; - optional uint64 parallel_persistence_buffer = 2; } message Postgres { @@ -16,4 +17,6 @@ message SnapshotRecovery { optional Postgres postgres = 2; optional Tree tree = 3; optional uint32 l1_batch = 4; + optional config.object_store.ObjectStore object_store = 5; + optional experimental.SnapshotRecovery experimental = 6; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 237f6057d794..ea16d1b158b4 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -10,26 +10,6 @@ use zksync_protobuf::ProtoRepr; use crate::{proto::snapshot_recovery as proto, read_optional_repr}; -impl ProtoRepr for proto::Tree { - type Type = TreeRecoveryConfig; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - chunk_size: self.chunk_size, - parallel_persistence_buffer: self - .parallel_persistence_buffer - .and_then(|a| NonZeroUsize::new(a as usize)), - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - chunk_size: this.chunk_size, - parallel_persistence_buffer: this.parallel_persistence_buffer.map(|a| a.get() as u64), - } - } -} - impl ProtoRepr for proto::Postgres { type Type = PostgresRecoveryConfig; @@ -52,11 +32,29 @@ impl ProtoRepr for proto::SnapshotRecovery { type Type = SnapshotRecoveryConfig; fn read(&self) -> anyhow::Result { + let tree = self + .tree + .as_ref() + .map(|tree| { + let chunk_size = tree.chunk_size; + let parallel_persistence_buffer = self + .experimental + .as_ref() + .and_then(|a| { + a.tree_recovery_parallel_persistence_buffer + .map(|a| NonZeroUsize::new(a as usize)) + }) + .flatten(); + TreeRecoveryConfig { + chunk_size, + parallel_persistence_buffer, + } + }) + .unwrap_or_default(); + Ok(Self::Type { enabled: self.enabled.unwrap_or_default(), - tree: read_optional_repr(&self.tree) - .context("tree")? - .unwrap_or_default(), + tree, postgres: read_optional_repr(&self.postgres) .context("postgres")? .unwrap_or_default(), @@ -65,10 +63,20 @@ impl ProtoRepr for proto::SnapshotRecovery { } fn build(this: &Self::Type) -> Self { - let tree = if this.tree == TreeRecoveryConfig::default() { - None + let (tree, experimental) = if this.tree == TreeRecoveryConfig::default() { + (None, None) } else { - Some(this.tree.clone()) + ( + Some(proto::Tree { + chunk_size: this.tree.chunk_size, + }), + Some(crate::proto::experimental::SnapshotRecovery { + tree_recovery_parallel_persistence_buffer: this + .tree + .parallel_persistence_buffer + .map(|a| a.get() as u64), + }), + ) }; let postgres = if this.postgres == PostgresRecoveryConfig::default() { None @@ -78,7 +86,8 @@ impl ProtoRepr for proto::SnapshotRecovery { Self { enabled: Some(this.enabled), postgres: postgres.as_ref().map(ProtoRepr::build), - tree: tree.as_ref().map(ProtoRepr::build), + tree, + experimental, l1_batch: this.l1_batch.map(|a| a.0), } } From 3fd0e5e559701dada39ccd52b03d77b24e9896e0 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 17:26:55 +0200 Subject: [PATCH 33/56] Fix object store Signed-off-by: Danil --- core/lib/config/src/configs/snapshot_recovery.rs | 3 +++ core/lib/protobuf_config/src/snapshot_recovery.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index ef2ba49011dd..ba26583a8a63 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -3,6 +3,8 @@ use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::L1BatchNumber; +use crate::ObjectStoreConfig; + #[derive(Debug, Clone, PartialEq, Deserialize, Default)] pub struct TreeRecoveryConfig { /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. @@ -38,4 +40,5 @@ pub struct SnapshotRecoveryConfig { pub l1_batch: Option, pub tree: TreeRecoveryConfig, pub postgres: PostgresRecoveryConfig, + pub object_store: Option, } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index ea16d1b158b4..4023cbb0c097 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -59,6 +59,7 @@ impl ProtoRepr for proto::SnapshotRecovery { .context("postgres")? .unwrap_or_default(), l1_batch: self.l1_batch.map(L1BatchNumber), + object_store: read_optional_repr(&self.object_store).context("object store")?, }) } @@ -89,6 +90,7 @@ impl ProtoRepr for proto::SnapshotRecovery { tree, experimental, l1_batch: this.l1_batch.map(|a| a.0), + object_store: this.object_store.as_ref().map(ProtoRepr::build), } } } From 506bbdcfbb609a209f2965763e7cdb00bd660430 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 17:32:57 +0200 Subject: [PATCH 34/56] Use derive from strum Signed-off-by: Danil --- Cargo.lock | 1 - Cargo.toml | 1 - core/node/api_server/Cargo.toml | 3 +-- core/node/api_server/src/web3/mod.rs | 2 +- etc/env/file_based/general.yaml | 3 ++- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9480bd3a7a0c..6800b5f5f8f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8640,7 +8640,6 @@ dependencies = [ "serde", "serde_json", "strum", - "strum_macros", "test-casing", "thiserror", "thread_local", diff --git a/Cargo.toml b/Cargo.toml index 216030b56a2c..5d9f6adf37ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,6 @@ sqlx = "0.7.3" static_assertions = "1.1" structopt = "0.3.20" strum = "0.24" -strum_macros = "0.24" tempfile = "3.0.2" test-casing = "0.1.2" test-log = "0.2.15" diff --git a/core/node/api_server/Cargo.toml b/core/node/api_server/Cargo.toml index 6f0fad424a22..da872ceb41ea 100644 --- a/core/node/api_server/Cargo.toml +++ b/core/node/api_server/Cargo.toml @@ -48,8 +48,7 @@ pin-project-lite.workspace = true hex.workspace = true http.workspace = true tower.workspace = true -strum.workspace = true -strum_macros.workspace = true +strum = { workspace = true, features = ["derive"] } tower-http = { workspace = true, features = ["cors", "metrics"] } lru.workspace = true diff --git a/core/node/api_server/src/web3/mod.rs b/core/node/api_server/src/web3/mod.rs index 32ca1816254d..7b2dec7abb35 100644 --- a/core/node/api_server/src/web3/mod.rs +++ b/core/node/api_server/src/web3/mod.rs @@ -86,7 +86,7 @@ enum ApiTransport { Http(SocketAddr), } -#[derive(Debug, Deserialize, Clone, PartialEq, strum_macros::EnumString)] +#[derive(Debug, Deserialize, Clone, PartialEq, strum::EnumString)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum Namespace { diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 522fde93a1f9..c03c36f28da2 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -339,7 +339,8 @@ snapshot_recovery: max_concurrency: 10 tree: chunk_size: 200000 - parallel_persistence_buffer: 1 + experimental: + tree_recovery_parallel_persistence_buffer: 1 pruning: enabled: true chunk_size: 10 From be4b9dcb526c000987b43da70798b8d72e76d6f7 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 15:15:28 +0200 Subject: [PATCH 35/56] Add external node config Signed-off-by: Danil --- zk_toolbox/crates/config/src/chain.rs | 3 +++ zk_toolbox/crates/config/src/consts.rs | 2 ++ zk_toolbox/crates/config/src/ecosystem.rs | 1 + zk_toolbox/crates/config/src/external_node.rs | 22 ++++++++++++++++ zk_toolbox/crates/config/src/lib.rs | 25 ++++++++++--------- .../zk_inception/src/commands/chain/create.rs | 1 + 6 files changed, 42 insertions(+), 12 deletions(-) create mode 100644 zk_toolbox/crates/config/src/external_node.rs diff --git a/zk_toolbox/crates/config/src/chain.rs b/zk_toolbox/crates/config/src/chain.rs index e685b0966b44..6cde25537879 100644 --- a/zk_toolbox/crates/config/src/chain.rs +++ b/zk_toolbox/crates/config/src/chain.rs @@ -30,6 +30,7 @@ pub struct ChainConfigInternal { pub prover_version: ProverMode, pub configs: PathBuf, pub rocks_db_path: PathBuf, + pub external_node_config_path: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -47,6 +48,7 @@ pub struct ChainConfig { pub link_to_code: PathBuf, pub rocks_db_path: PathBuf, pub configs: PathBuf, + pub external_node_config_path: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, pub base_token: BaseToken, pub wallet_creation: WalletCreation, @@ -113,6 +115,7 @@ impl ChainConfig { prover_version: self.prover_version, configs: self.configs.clone(), rocks_db_path: self.rocks_db_path.clone(), + external_node_config_path: self.external_node_config_path.clone(), l1_batch_commit_data_generator_mode: self.l1_batch_commit_data_generator_mode, base_token: self.base_token.clone(), wallet_creation: self.wallet_creation, diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 9141d044af94..a00274fb35f3 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -11,6 +11,8 @@ pub(crate) const GENERAL_FILE: &str = "general.yaml"; /// Name of the genesis config file pub(crate) const GENESIS_FILE: &str = "genesis.yaml"; +// Name of external node specific config +pub(crate) const EN_CONFIG_FILE: &str = "external_node.yaml"; pub(crate) const ERC20_CONFIGS_FILE: &str = "erc20.yaml"; /// Name of the initial deployments config file pub(crate) const INITIAL_DEPLOYMENT_FILE: &str = "initial_deployments.yaml"; diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 1557ab21646f..08708ebb0b61 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -120,6 +120,7 @@ impl EcosystemConfig { chain_id: config.chain_id, prover_version: config.prover_version, configs: config.configs, + external_node_config_path: config.external_node_config_path, l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode, l1_network: self.l1_network, link_to_code: self diff --git a/zk_toolbox/crates/config/src/external_node.rs b/zk_toolbox/crates/config/src/external_node.rs new file mode 100644 index 000000000000..d5f94daac59f --- /dev/null +++ b/zk_toolbox/crates/config/src/external_node.rs @@ -0,0 +1,22 @@ +use std::num::NonZeroUsize; + +use serde::Deserialize; +use types::{ChainId, L1BatchCommitDataGeneratorMode}; + +use crate::{consts::EN_CONFIG_FILE, traits::FileConfigWithDefaultName}; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ENConfig { + // Genesis + pub l2_chain_id: ChainId, + pub l1_chain_id: ChainId, + pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, + + // Main node configuration + pub main_node_url: String, + pub main_node_rate_limit_rps: Option, +} + +impl FileConfigWithDefaultName for ENConfig { + const FILE_NAME: &'static str = EN_CONFIG_FILE; +} diff --git a/zk_toolbox/crates/config/src/lib.rs b/zk_toolbox/crates/config/src/lib.rs index 8e40da7bf6bd..a80a2b6fe5de 100644 --- a/zk_toolbox/crates/config/src/lib.rs +++ b/zk_toolbox/crates/config/src/lib.rs @@ -1,3 +1,15 @@ +pub use chain::*; +pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; +pub use contracts::*; +pub use ecosystem::*; +pub use file_config::*; +pub use general::*; +pub use genesis::*; +pub use manipulations::*; +pub use secrets::*; +pub use wallet_creation::*; +pub use wallets::*; + mod chain; mod consts; mod contracts; @@ -10,17 +22,6 @@ mod secrets; mod wallet_creation; mod wallets; +pub mod external_node; pub mod forge_interface; pub mod traits; - -pub use chain::*; -pub use consts::{DOCKER_COMPOSE_FILE, ZKSYNC_ERA_GIT_REPO}; -pub use contracts::*; -pub use ecosystem::*; -pub use file_config::*; -pub use general::*; -pub use genesis::*; -pub use manipulations::*; -pub use secrets::*; -pub use wallet_creation::*; -pub use wallets::*; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs index f915a3b8d6f6..dc8f408db3b3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/create.rs @@ -68,6 +68,7 @@ pub(crate) fn create_chain_inner( link_to_code: ecosystem_config.link_to_code.clone(), rocks_db_path: ecosystem_config.get_chain_rocks_db_path(&default_chain_name), configs: chain_configs_path.clone(), + external_node_config_path: None, l1_batch_commit_data_generator_mode: args.l1_batch_commit_data_generator_mode, base_token: args.base_token, wallet_creation: args.wallet_creation, From af5e0ed1cdfa4ff36b80457d5e3f157a5bcb485a Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 18:03:20 +0200 Subject: [PATCH 36/56] Generate en configs Signed-off-by: Danil --- chains/era/ZkStack.yaml | 1 + zk_toolbox/crates/config/src/chain.rs | 9 ++- zk_toolbox/crates/config/src/external_node.rs | 7 +- zk_toolbox/crates/config/src/general.rs | 66 +++++++++++++++++++ .../zk_inception/src/commands/chain/mod.rs | 17 +++-- .../src/commands/chain/prepare_en_config.rs | 59 +++++++++++++++++ .../crates/zk_inception/src/messages.rs | 7 ++ 7 files changed, 154 insertions(+), 12 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs diff --git a/chains/era/ZkStack.yaml b/chains/era/ZkStack.yaml index 17b307cac4f6..8dbd49c02c67 100644 --- a/chains/era/ZkStack.yaml +++ b/chains/era/ZkStack.yaml @@ -4,6 +4,7 @@ chain_id: 271 prover_version: NoProofs configs: ./chains/era/configs/ rocks_db_path: ./chains/era/db/ +external_node_config_path: ./chains/era/configs/external_node l1_batch_commit_data_generator_mode: Rollup base_token: address: '0x0000000000000000000000000000000000000001' diff --git a/zk_toolbox/crates/config/src/chain.rs b/zk_toolbox/crates/config/src/chain.rs index 6cde25537879..a2296d9ca166 100644 --- a/zk_toolbox/crates/config/src/chain.rs +++ b/zk_toolbox/crates/config/src/chain.rs @@ -11,11 +11,12 @@ use xshell::Shell; use crate::{ consts::{ - CONFIG_NAME, CONTRACTS_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, SECRETS_FILE, WALLETS_FILE, + CONFIG_NAME, CONTRACTS_FILE, GENERAL_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, + SECRETS_FILE, WALLETS_FILE, }, create_localhost_wallets, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, - ContractsConfig, GenesisConfig, SecretsConfig, WalletsConfig, + ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig, }; /// Chain configuration file. This file is created in the chain @@ -73,6 +74,10 @@ impl ChainConfig { GenesisConfig::read(self.get_shell(), self.configs.join(GENESIS_FILE)) } + pub fn get_general_config(&self) -> anyhow::Result { + GeneralConfig::read(self.get_shell(), self.configs.join(GENERAL_FILE)) + } + pub fn get_wallets_config(&self) -> anyhow::Result { let path = self.configs.join(WALLETS_FILE); if let Ok(wallets) = WalletsConfig::read(self.get_shell(), &path) { diff --git a/zk_toolbox/crates/config/src/external_node.rs b/zk_toolbox/crates/config/src/external_node.rs index d5f94daac59f..87acb15e4d8c 100644 --- a/zk_toolbox/crates/config/src/external_node.rs +++ b/zk_toolbox/crates/config/src/external_node.rs @@ -1,19 +1,20 @@ use std::num::NonZeroUsize; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use types::{ChainId, L1BatchCommitDataGeneratorMode}; use crate::{consts::EN_CONFIG_FILE, traits::FileConfigWithDefaultName}; -#[derive(Debug, Clone, PartialEq, Deserialize)] +#[derive(Debug, Clone, PartialEq, Deserialize, Serialize)] pub struct ENConfig { // Genesis pub l2_chain_id: ChainId, - pub l1_chain_id: ChainId, + pub l1_chain_id: u32, pub l1_batch_commit_data_generator_mode: L1BatchCommitDataGeneratorMode, // Main node configuration pub main_node_url: String, + #[serde(skip_serializing_if = "Option::is_none")] pub main_node_rate_limit_rps: Option, } diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 058f23bf1b5d..1fcc22801474 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -8,6 +8,7 @@ use crate::{consts::GENERAL_FILE, traits::FileConfigWithDefaultName}; pub struct GeneralConfig { pub db: RocksDBConfig, pub eth: EthConfig, + pub api: ApiConfig, #[serde(flatten)] pub other: serde_json::Value, } @@ -45,3 +46,68 @@ pub struct EthSender { #[serde(flatten)] pub other: serde_json::Value, } + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct ApiConfig { + /// Configuration options for the Web3 JSON RPC servers. + pub web3_json_rpc: Web3JsonRpcConfig, + /// Configuration options for the Prometheus exporter. + pub prometheus: PrometheusConfig, + /// Configuration options for the Health check. + pub healthcheck: HealthCheckConfig, + /// Configuration options for Merkle tree API. + pub merkle_tree: MerkleTreeApiConfig, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct Web3JsonRpcConfig { + /// Port to which the HTTP RPC server is listening. + pub http_port: u16, + /// URL to access HTTP RPC server. + pub http_url: String, + /// Port to which the WebSocket RPC server is listening. + pub ws_port: u16, + /// URL to access WebSocket RPC server. + pub ws_url: String, + /// Max possible limit of entities to be requested once. + pub req_entities_limit: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct PrometheusConfig { + /// Port to which the Prometheus exporter server is listening. + pub listener_port: u16, + /// URL of the push gateway. + pub pushgateway_url: String, + /// Push interval in ms. + pub push_interval_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct HealthCheckConfig { + /// Port to which the REST server is listening. + pub port: u16, + /// Time limit in milliseconds to mark a health check as slow and log the corresponding warning. + /// If not specified, the default value in the health check crate will be used. + pub slow_time_limit_ms: Option, + /// Time limit in milliseconds to abort a health check and return "not ready" status for the corresponding component. + /// If not specified, the default value in the health check crate will be used. + pub hard_time_limit_ms: Option, + #[serde(flatten)] + pub other: serde_json::Value, +} + +/// Configuration for the Merkle tree API. +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct MerkleTreeApiConfig { + /// Port to bind the Merkle tree API server to. + pub port: u16, + #[serde(flatten)] + pub other: serde_json::Value, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index 759b4aaea557..d55c53beee10 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -1,10 +1,3 @@ -pub(crate) mod args; -mod create; -pub mod deploy_paymaster; -pub mod genesis; -pub(crate) mod init; -mod initialize_bridges; - pub(crate) use args::create::ChainCreateArgsFinal; use clap::Subcommand; use common::forge::ForgeScriptArgs; @@ -13,6 +6,14 @@ use xshell::Shell; use crate::commands::chain::args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}; +pub(crate) mod args; +mod create; +pub mod deploy_paymaster; +pub mod genesis; +pub(crate) mod init; +mod initialize_bridges; +mod prepare_en_config; + #[derive(Subcommand, Debug)] pub enum ChainCommands { /// Create a new chain, setting the necessary configurations for later initialization @@ -25,6 +26,7 @@ pub enum ChainCommands { InitializeBridges(ForgeScriptArgs), /// Initialize bridges on l2 DeployPaymaster(ForgeScriptArgs), + PrepareExternalNodeConfig, } pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<()> { @@ -34,5 +36,6 @@ pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<() ChainCommands::Genesis(args) => genesis::run(args, shell).await, ChainCommands::InitializeBridges(args) => initialize_bridges::run(args, shell).await, ChainCommands::DeployPaymaster(args) => deploy_paymaster::run(args, shell).await, + ChainCommands::PrepareExternalNodeConfig => prepare_en_config::run(shell), } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs new file mode 100644 index 000000000000..cdb54d80524d --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs @@ -0,0 +1,59 @@ +use std::path::Path; + +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ + external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig, +}; +use xshell::Shell; + +use crate::messages::{ + msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, +}; + +pub fn run(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_PREPARING_EN_CONFIGS); + let chain_name = global_config().chain_name.clone(); + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let mut chain_config = ecosystem_config + .load_chain(chain_name) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + let external_node_config_path = chain_config + .external_node_config_path + .unwrap_or_else(|| chain_config.configs.join("external_node")); + shell.create_dir(&external_node_config_path)?; + chain_config.external_node_config_path = Some(external_node_config_path.clone()); + prepare_configs(shell, &chain_config, &external_node_config_path)?; + let chain_path = ecosystem_config.chains.join(&chain_config.name); + chain_config.save_with_base_path(shell, chain_path)?; + logger::info(msg_preparing_en_config_is_done(&external_node_config_path)); + Ok(()) +} + +fn prepare_configs( + shell: &Shell, + config: &ChainConfig, + en_configs_path: &Path, +) -> anyhow::Result<()> { + let genesis = config.get_genesis_config()?; + let general = config.get_general_config()?; + let en_config = ENConfig { + l2_chain_id: genesis.l2_chain_id, + l1_chain_id: genesis.l1_chain_id, + l1_batch_commit_data_generator_mode: genesis + .l1_batch_commit_data_generator_mode + .unwrap_or_default(), + main_node_url: general.api.web3_json_rpc.http_url.clone(), + main_node_rate_limit_rps: None, + }; + let mut general_en = general.clone(); + general_en.api.web3_json_rpc.http_port = general_en.api.web3_json_rpc.http_port + 1000; + general_en.api.web3_json_rpc.ws_port = general_en.api.web3_json_rpc.ws_port + 1000; + general_en.api.healthcheck.port = general_en.api.healthcheck.port + 1000; + general_en.api.merkle_tree.port = general_en.api.merkle_tree.port + 1000; + general_en.api.prometheus.listener_port = general_en.api.prometheus.listener_port + 1000; + general_en.save_with_base_path(shell, &en_configs_path)?; + en_config.save_with_base_path(shell, &en_configs_path)?; + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 1b3c05258753..64cb32bcf9f4 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use ethers::{ types::{H160, U256}, utils::format_ether, @@ -173,6 +175,7 @@ pub(super) const MSG_FAILED_TO_FIND_ECOSYSTEM_ERR: &str = "Failed to find ecosys pub(super) const MSG_STARTING_SERVER: &str = "Starting server"; pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts..."; +pub(super) const MSG_PREPARING_EN_CONFIGS: &str = "Preparing External Node config"; /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; @@ -189,6 +192,10 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( ) } +pub(super) fn msg_preparing_en_config_is_done(path: &Path) -> String { + format!("External nodes configs could be found in: {path:?}") +} + /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; From a04cd46a8c1589ab3a79c91c6db208245a1fc8cf Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 18:38:10 +0200 Subject: [PATCH 37/56] Run external node Signed-off-by: Danil --- .../zk_inception/src/commands/args/mod.rs | 6 +- .../src/commands/args/run_external_node.rs | 13 ++++ .../src/commands/args/run_server.rs | 2 +- .../src/commands/chain/genesis.rs | 2 +- .../zk_inception/src/commands/chain/init.rs | 19 +++-- .../src/commands/external_node.rs | 34 ++++++++ .../crates/zk_inception/src/commands/mod.rs | 1 + .../zk_inception/src/commands/server.rs | 17 +--- .../crates/zk_inception/src/external_node.rs | 77 +++++++++++++++++++ zk_toolbox/crates/zk_inception/src/main.rs | 8 +- zk_toolbox/crates/zk_inception/src/server.rs | 8 +- 11 files changed, 162 insertions(+), 25 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node.rs create mode 100644 zk_toolbox/crates/zk_inception/src/external_node.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs index bf1457ba92c6..0bb080cc6dc8 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs @@ -1,3 +1,5 @@ -mod run_server; - +pub use run_external_node::*; pub use run_server::*; + +mod run_external_node; +mod run_server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs new file mode 100644 index 000000000000..22cf218b9e50 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs @@ -0,0 +1,13 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use crate::messages::{MSG_SERVER_ADDITIONAL_ARGS_HELP, MSG_SERVER_COMPONENTS_HELP}; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct RunExternalNodeArgs { + #[clap(long, help = MSG_SERVER_COMPONENTS_HELP)] + pub components: Option>, + #[clap(long, short)] + #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] + pub additional_args: Vec, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs index 1ec211c25f6d..74bafd6ce5ef 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs @@ -13,5 +13,5 @@ pub struct RunServerArgs { pub genesis: bool, #[clap(long, short)] #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] - additional_args: Vec, + pub additional_args: Vec, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 8c4edc88290d..4baf78d22fe0 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -128,5 +128,5 @@ async fn initialize_databases( fn run_server_genesis(chain_config: &ChainConfig, shell: &Shell) -> anyhow::Result<()> { let server = RunServer::new(None, chain_config); - server.run(shell, ServerMode::Genesis) + server.run(shell, ServerMode::Genesis, vec![]) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 0c9ac8743eee..bb7b60e16738 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -1,5 +1,6 @@ use anyhow::Context; use common::{ + cmd::Cmd, config::global_config, forge::{Forge, ForgeScriptArgs}, logger, @@ -14,7 +15,7 @@ use config::{ traits::{ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath}, ChainConfig, ContractsConfig, EcosystemConfig, }; -use xshell::Shell; +use xshell::{cmd, Shell}; use super::args::init::InitArgsFinal; use crate::{ @@ -25,9 +26,9 @@ use crate::{ config_manipulations::{update_genesis, update_l1_contracts, update_l1_rpc_url_secret}, forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ - msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, - MSG_CHAIN_NOT_FOUND_ERR, MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, - MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, + msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, + MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR, + MSG_GENESIS_DATABASE_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, }, }; @@ -55,7 +56,7 @@ pub async fn init( chain_config: &ChainConfig, ) -> anyhow::Result<()> { copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; - + build_l1_contracts(shell, ecosystem_config)?; update_genesis(shell, chain_config)?; update_l1_rpc_url_secret(shell, chain_config, init_args.l1_rpc_url.clone())?; let mut contracts_config = @@ -138,3 +139,11 @@ async fn register_chain( )?; update_l1_contracts(shell, chain_config, ®ister_chain_output) } + +fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); + let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); + Cmd::new(cmd!(shell, "yarn build")).run()?; + spinner.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs new file mode 100644 index 000000000000..94363168d46b --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs @@ -0,0 +1,34 @@ +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::Shell; + +use crate::{ + commands::args::RunExternalNodeArgs, + external_node::RunExternalNode, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, +}; + +pub fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + logger::info(MSG_STARTING_SERVER); + + run_external_node(args, &chain_config, shell)?; + + Ok(()) +} + +fn run_external_node( + args: RunExternalNodeArgs, + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result<()> { + let server = RunExternalNode::new(args.components.clone(), chain_config)?; + server.run(shell, args.additional_args.clone()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index ccdf5b082caa..10fccd316eee 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -3,4 +3,5 @@ pub mod chain; pub mod containers; pub mod ecosystem; pub mod prover; +pub mod external_node; pub mod server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/server.rs b/zk_toolbox/crates/zk_inception/src/commands/server.rs index e2d35dd9b792..aed16357c925 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/server.rs @@ -1,11 +1,11 @@ use anyhow::Context; -use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; +use common::{config::global_config, logger}; use config::{ChainConfig, EcosystemConfig}; -use xshell::{cmd, Shell}; +use xshell::Shell; use crate::{ commands::args::RunServerArgs, - messages::{MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, server::{RunServer, ServerMode}, }; @@ -19,20 +19,11 @@ pub fn run(shell: &Shell, args: RunServerArgs) -> anyhow::Result<()> { logger::info(MSG_STARTING_SERVER); - build_l1_contracts(shell, &ecosystem_config)?; run_server(args, &chain_config, shell)?; Ok(()) } -fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { - let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); - let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); - Cmd::new(cmd!(shell, "yarn build")).run()?; - spinner.finish(); - Ok(()) -} - fn run_server( args: RunServerArgs, chain_config: &ChainConfig, @@ -44,5 +35,5 @@ fn run_server( } else { ServerMode::Normal }; - server.run(shell, mode) + server.run(shell, mode, args.additional_args) } diff --git a/zk_toolbox/crates/zk_inception/src/external_node.rs b/zk_toolbox/crates/zk_inception/src/external_node.rs new file mode 100644 index 000000000000..baf00cccae5f --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/external_node.rs @@ -0,0 +1,77 @@ +use std::path::PathBuf; + +use anyhow::Context; +use common::cmd::Cmd; +use config::{ + external_node::ENConfig, traits::FileConfigWithDefaultName, ChainConfig, GeneralConfig, + SecretsConfig, +}; +use xshell::{cmd, Shell}; + +use crate::messages::MSG_FAILED_TO_RUN_SERVER_ERR; + +pub struct RunExternalNode { + components: Option>, + code_path: PathBuf, + general_config: PathBuf, + secrets: PathBuf, + en_config: PathBuf, +} + +impl RunExternalNode { + pub fn new( + components: Option>, + chain_config: &ChainConfig, + ) -> anyhow::Result { + let en_path = chain_config + .external_node_config_path + .clone() + .context("External node is not initialized")?; + let general_config = GeneralConfig::get_path_with_base_path(&en_path); + let secrets = SecretsConfig::get_path_with_base_path(&en_path); + let enconfig = ENConfig::get_path_with_base_path(&en_path); + + Ok(Self { + components, + code_path: chain_config.link_to_code.clone(), + general_config, + secrets, + en_config: enconfig, + }) + } + + pub fn run(&self, shell: &Shell, mut additional_args: Vec) -> anyhow::Result<()> { + shell.change_dir(&self.code_path); + let config_general_config = &self.general_config.to_str().unwrap(); + let en_config = &self.en_config.to_str().unwrap(); + let secrets = &self.secrets.to_str().unwrap(); + if let Some(components) = self.components() { + additional_args.push(format!("--components={}", components)) + } + let mut cmd = Cmd::new( + cmd!( + shell, + "cargo run --release --bin zksync_external_node -- + --config-path {config_general_config} + --secrets-path {secrets} + --external-node-config-path {en_config} + " + ) + .args(additional_args) + .env_remove("RUSTUP_TOOLCHAIN"), + ) + .with_force_run(); + + cmd.run().context(MSG_FAILED_TO_RUN_SERVER_ERR)?; + Ok(()) + } + + fn components(&self) -> Option { + self.components.as_ref().and_then(|components| { + if components.is_empty() { + return None; + } + Some(components.join(",")) + }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index dff9e479e01f..bdf9fe9fd78a 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -8,7 +8,9 @@ use config::EcosystemConfig; use xshell::Shell; use crate::commands::{ - args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, prover::ProverCommands, + args::{RunExternalNodeArgs, RunServerArgs}, + chain::ChainCommands, + ecosystem::EcosystemCommands, }; pub mod accept_ownership; @@ -16,6 +18,7 @@ mod commands; mod config_manipulations; mod consts; mod defaults; +pub mod external_node; pub mod forge_utils; mod messages; pub mod server; @@ -42,6 +45,8 @@ pub enum InceptionSubcommands { Prover(ProverCommands), /// Run server Server(RunServerArgs), + // Run External Node + ExternalNode(RunExternalNodeArgs), /// Run containers for local development Containers, } @@ -109,6 +114,7 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, + InceptionSubcommands::ExternalNode(args) => commands::external_node::run(shell, args)?, } Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/server.rs b/zk_toolbox/crates/zk_inception/src/server.rs index 6773d224cba3..c4feb1c7c272 100644 --- a/zk_toolbox/crates/zk_inception/src/server.rs +++ b/zk_toolbox/crates/zk_inception/src/server.rs @@ -44,14 +44,18 @@ impl RunServer { } } - pub fn run(&self, shell: &Shell, server_mode: ServerMode) -> anyhow::Result<()> { + pub fn run( + &self, + shell: &Shell, + server_mode: ServerMode, + mut additional_args: Vec, + ) -> anyhow::Result<()> { shell.change_dir(&self.code_path); let config_genesis = &self.genesis.to_str().unwrap(); let config_wallets = &self.wallets.to_str().unwrap(); let config_general_config = &self.general_config.to_str().unwrap(); let config_contracts = &self.contracts.to_str().unwrap(); let secrets = &self.secrets.to_str().unwrap(); - let mut additional_args = vec![]; if let Some(components) = self.components() { additional_args.push(format!("--components={}", components)) } From b364d23eebad2d9a7c5f7122e3c879de6aa166dd Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 20:11:07 +0200 Subject: [PATCH 38/56] Add secrets Signed-off-by: Danil --- zk_toolbox/crates/config/src/secrets.rs | 3 +- .../src/commands/args/run_external_node.rs | 2 + .../src/commands/chain/args/genesis.rs | 6 +- .../src/commands/chain/args/mod.rs | 1 + .../args/prepare_external_node_configs.rs | 65 +++++++++++++++++++ .../src/commands/chain/genesis.rs | 4 +- .../zk_inception/src/commands/chain/mod.rs | 9 ++- .../src/commands/chain/prepare_en_config.rs | 31 +++++++-- .../src/commands/external_node.rs | 35 ++++++++-- .../zk_inception/src/config_manipulations.rs | 2 +- zk_toolbox/crates/zk_inception/src/consts.rs | 2 + .../crates/zk_inception/src/defaults.rs | 8 +++ zk_toolbox/crates/zk_inception/src/main.rs | 4 +- .../crates/zk_inception/src/messages.rs | 10 ++- zk_toolbox/crates/zk_supervisor/src/dals.rs | 2 +- 15 files changed, 159 insertions(+), 25 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index ebacc5d437cb..86d0585e354d 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -6,7 +6,8 @@ use crate::{consts::SECRETS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct DatabaseSecrets { pub server_url: Url, - pub prover_url: Url, + #[serde(skip_serializing_if = "Option::is_none")] + pub prover_url: Option, #[serde(flatten)] pub other: serde_json::Value, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs index 22cf218b9e50..1bc0c06728d7 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs @@ -5,6 +5,8 @@ use crate::messages::{MSG_SERVER_ADDITIONAL_ARGS_HELP, MSG_SERVER_COMPONENTS_HEL #[derive(Debug, Serialize, Deserialize, Parser)] pub struct RunExternalNodeArgs { + #[clap(long)] + pub reinit: bool, #[clap(long, help = MSG_SERVER_COMPONENTS_HELP)] pub components: Option>, #[clap(long, short)] diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs index d835b1eb36a6..e3507fdca41d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs @@ -8,8 +8,8 @@ use crate::{ defaults::{generate_db_names, DBNames, DATABASE_PROVER_URL, DATABASE_SERVER_URL}, messages::{ msg_prover_db_name_prompt, msg_prover_db_url_prompt, msg_server_db_name_prompt, - msg_server_db_url_prompt, MSG_GENESIS_USE_DEFAULT_HELP, MSG_PROVER_DB_NAME_HELP, - MSG_PROVER_DB_URL_HELP, MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, + msg_server_db_url_prompt, MSG_PROVER_DB_NAME_HELP, MSG_PROVER_DB_URL_HELP, + MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, MSG_USE_DEFAULT_DATABASES_HELP, }, }; @@ -23,7 +23,7 @@ pub struct GenesisArgs { pub prover_db_url: Option, #[clap(long, help = MSG_PROVER_DB_NAME_HELP)] pub prover_db_name: Option, - #[clap(long, short, help = MSG_GENESIS_USE_DEFAULT_HELP)] + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] pub use_default: bool, #[clap(long, short, action)] pub dont_drop: bool, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs index 08f39a90a843..4c5c1e1d4f74 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs @@ -1,3 +1,4 @@ pub mod create; pub mod genesis; pub mod init; +pub mod prepare_external_node_configs; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs new file mode 100644 index 000000000000..bcfd5769a370 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs @@ -0,0 +1,65 @@ +use clap::Parser; +use common::{db::DatabaseConfig, slugify, Prompt}; +use config::ChainConfig; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::{ + defaults::{generate_external_node_db_name, DATABASE_SERVER_URL, LOCAL_RPC_URL}, + messages::{ + msg_external_node_db_name_prompt, msg_external_node_db_url_prompt, MSG_L1_RPC_URL_PROMPT, + MSG_USE_DEFAULT_DATABASES_HELP, + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Parser, Default)] +pub struct PrepareConfigArgs { + #[clap(long)] + pub db_url: Option, + #[clap(long)] + pub db_name: Option, + #[clap(long)] + pub l1_rpc_url: Option, + #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] + pub use_default: bool, +} + +impl PrepareConfigArgs { + pub fn fill_values_with_prompt(self, config: &ChainConfig) -> PrepareConfigFinal { + let db_name = generate_external_node_db_name(config); + let chain_name = config.name.clone(); + if self.use_default { + PrepareConfigFinal { + db: DatabaseConfig::new(DATABASE_SERVER_URL.clone(), db_name), + l1_rpc_url: LOCAL_RPC_URL.to_string(), + } + } else { + let db_url = self.db_url.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_url_prompt(&chain_name)) + .default(DATABASE_SERVER_URL.as_str()) + .ask() + }); + let db_name = slugify(&self.db_name.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_name_prompt(&chain_name)) + .default(&db_name) + .ask() + })); + let l1_rpc_url = self.l1_rpc_url.unwrap_or_else(|| { + Prompt::new(&MSG_L1_RPC_URL_PROMPT) + .default(&LOCAL_RPC_URL) + .ask() + }); + + PrepareConfigFinal { + db: DatabaseConfig::new(db_url, db_name), + l1_rpc_url, + } + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PrepareConfigFinal { + pub db: DatabaseConfig, + pub l1_rpc_url: String, +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 4baf78d22fe0..9021cb3edc3d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -14,6 +14,7 @@ use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, config_manipulations::{update_database_secrets, update_general_config}, + consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, @@ -24,9 +25,6 @@ use crate::{ server::{RunServer, ServerMode}, }; -const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; -const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; - pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index d55c53beee10..f2478111e3d3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -4,7 +4,10 @@ use common::forge::ForgeScriptArgs; pub(crate) use create::create_chain_inner; use xshell::Shell; -use crate::commands::chain::args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}; +use crate::commands::chain::args::{ + create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs, + prepare_external_node_configs::PrepareConfigArgs, +}; pub(crate) mod args; mod create; @@ -26,7 +29,7 @@ pub enum ChainCommands { InitializeBridges(ForgeScriptArgs), /// Initialize bridges on l2 DeployPaymaster(ForgeScriptArgs), - PrepareExternalNodeConfig, + PrepareExternalNodeConfig(PrepareConfigArgs), } pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<()> { @@ -36,6 +39,6 @@ pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<() ChainCommands::Genesis(args) => genesis::run(args, shell).await, ChainCommands::InitializeBridges(args) => initialize_bridges::run(args, shell).await, ChainCommands::DeployPaymaster(args) => deploy_paymaster::run(args, shell).await, - ChainCommands::PrepareExternalNodeConfig => prepare_en_config::run(shell), + ChainCommands::PrepareExternalNodeConfig(args) => prepare_en_config::run(shell, args), } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs index cdb54d80524d..0a69ff0d48a1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs @@ -3,15 +3,19 @@ use std::path::Path; use anyhow::Context; use common::{config::global_config, logger}; use config::{ - external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig, + external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, DatabaseSecrets, + EcosystemConfig, L1Secret, SecretsConfig, }; use xshell::Shell; -use crate::messages::{ - msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, +use crate::{ + commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal}, + messages::{ + msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, + }, }; -pub fn run(shell: &Shell) -> anyhow::Result<()> { +pub fn run(shell: &Shell, args: PrepareConfigArgs) -> anyhow::Result<()> { logger::info(MSG_PREPARING_EN_CONFIGS); let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -19,12 +23,13 @@ pub fn run(shell: &Shell) -> anyhow::Result<()> { .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; + let args = args.fill_values_with_prompt(&chain_config); let external_node_config_path = chain_config .external_node_config_path .unwrap_or_else(|| chain_config.configs.join("external_node")); shell.create_dir(&external_node_config_path)?; chain_config.external_node_config_path = Some(external_node_config_path.clone()); - prepare_configs(shell, &chain_config, &external_node_config_path)?; + prepare_configs(shell, &chain_config, &external_node_config_path, args)?; let chain_path = ecosystem_config.chains.join(&chain_config.name); chain_config.save_with_base_path(shell, chain_path)?; logger::info(msg_preparing_en_config_is_done(&external_node_config_path)); @@ -35,6 +40,7 @@ fn prepare_configs( shell: &Shell, config: &ChainConfig, en_configs_path: &Path, + args: PrepareConfigFinal, ) -> anyhow::Result<()> { let genesis = config.get_genesis_config()?; let general = config.get_general_config()?; @@ -53,7 +59,22 @@ fn prepare_configs( general_en.api.healthcheck.port = general_en.api.healthcheck.port + 1000; general_en.api.merkle_tree.port = general_en.api.merkle_tree.port + 1000; general_en.api.prometheus.listener_port = general_en.api.prometheus.listener_port + 1000; + let secrets = SecretsConfig { + database: DatabaseSecrets { + server_url: args.db.full_url(), + prover_url: None, + other: Default::default(), + }, + l1: L1Secret { + l1_rpc_url: args.l1_rpc_url.clone(), + other: Default::default(), + }, + other: Default::default(), + }; + + secrets.save_with_base_path(shell, en_configs_path)?; general_en.save_with_base_path(shell, &en_configs_path)?; en_config.save_with_base_path(shell, &en_configs_path)?; + Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs index 94363168d46b..3fe7476d1c88 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs @@ -1,15 +1,22 @@ use anyhow::Context; -use common::{config::global_config, logger}; -use config::{ChainConfig, EcosystemConfig}; +use common::{ + config::global_config, + db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, + logger, +}; +use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; use xshell::Shell; use crate::{ commands::args::RunExternalNodeArgs, + consts::SERVER_MIGRATIONS, external_node::RunExternalNode, - messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_STARTING_SERVER, + }, }; -pub fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { +pub async fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; let chain = global_config().chain_name.clone(); @@ -19,16 +26,32 @@ pub fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { logger::info(MSG_STARTING_SERVER); - run_external_node(args, &chain_config, shell)?; + run_external_node(args, &chain_config, shell).await?; Ok(()) } -fn run_external_node( +async fn run_external_node( args: RunExternalNodeArgs, chain_config: &ChainConfig, shell: &Shell, ) -> anyhow::Result<()> { + if args.reinit { + let secrets = SecretsConfig::read_with_base_path( + shell, + chain_config + .external_node_config_path + .clone() + .context("External node is not initalized")?, + )?; + let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; + drop_db_if_exists(&db_config) + .await + .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; + init_db(&db_config).await?; + let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); + migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; + } let server = RunExternalNode::new(args.components.clone(), chain_config)?; server.run(shell, args.additional_args.clone()) } diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index a300a15e76c6..9885f4315549 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -31,7 +31,7 @@ pub(crate) fn update_database_secrets( ) -> anyhow::Result<()> { let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; secrets.database.server_url = server_db_config.full_url(); - secrets.database.prover_url = prover_db_config.full_url(); + secrets.database.prover_url = Some(prover_db_config.full_url()); secrets.save_with_base_path(shell, &config.configs)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index a59024d09b40..8dde9337a73f 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -1,3 +1,5 @@ pub const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; pub const MINIMUM_BALANCE_FOR_WALLET: u128 = 5000000000000000000; +pub const SERVER_MIGRATIONS: &str = "core/lib/dal/migrations"; +pub const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 04b735e02275..a0550c5b5efb 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -36,3 +36,11 @@ pub fn generate_db_names(config: &ChainConfig) -> DBNames { ), } } + +pub fn generate_external_node_db_name(config: &ChainConfig) -> String { + format!( + "external_node_{}_{}", + config.l1_network.to_string().to_ascii_lowercase(), + config.name + ) +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index bdf9fe9fd78a..d102e0d69a95 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -114,7 +114,9 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, - InceptionSubcommands::ExternalNode(args) => commands::external_node::run(shell, args)?, + InceptionSubcommands::ExternalNode(args) => { + commands::external_node::run(shell, args).await? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 64cb32bcf9f4..2114c1998f31 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -120,7 +120,7 @@ pub(super) const MSG_SERVER_DB_URL_HELP: &str = "Server database url without dat pub(super) const MSG_SERVER_DB_NAME_HELP: &str = "Server database name"; pub(super) const MSG_PROVER_DB_URL_HELP: &str = "Prover database url without database name"; pub(super) const MSG_PROVER_DB_NAME_HELP: &str = "Prover database name"; -pub(super) const MSG_GENESIS_USE_DEFAULT_HELP: &str = "Use default database urls and names"; +pub(super) const MSG_USE_DEFAULT_DATABASES_HELP: &str = "Use default database urls and names"; pub(super) const MSG_GENESIS_COMPLETED: &str = "Genesis completed successfully"; pub(super) const MSG_STARTING_GENESIS: &str = "Starting genesis process"; pub(super) const MSG_INITIALIZING_DATABASES_SPINNER: &str = "Initializing databases..."; @@ -135,6 +135,10 @@ pub(super) fn msg_server_db_url_prompt(chain_name: &str) -> String { format!("Please provide server database url for chain {chain_name}") } +pub(super) fn msg_external_node_db_url_prompt(chain_name: &str) -> String { + format!("Please provide external_node database url for chain {chain_name}") +} + pub(super) fn msg_prover_db_url_prompt(chain_name: &str) -> String { format!("Please provide prover database url for chain {chain_name}") } @@ -143,6 +147,10 @@ pub(super) fn msg_prover_db_name_prompt(chain_name: &str) -> String { format!("Please provide prover database name for chain {chain_name}") } +pub(super) fn msg_external_node_db_name_prompt(chain_name: &str) -> String { + format!("Please provide external_node database name for chain {chain_name}") +} + pub(super) fn msg_server_db_name_prompt(chain_name: &str) -> String { format!("Please provide server database name for chain {chain_name}") } diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index f2f6f86cfc61..7f53770bc012 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -46,7 +46,7 @@ pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { Ok(Dal { path: PROVER_DAL_PATH.to_string(), - url: secrets.database.prover_url.clone(), + url: secrets.database.prover_url.unwrap().clone(), }) } From 85bb8bd9b91d30832e46d427f46c088359cb8cb1 Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 19 Jun 2024 14:19:08 +0200 Subject: [PATCH 39/56] External node is running Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 14 ++++++++++--- core/bin/external_node/src/init.rs | 8 +++++--- core/bin/external_node/src/main.rs | 1 + etc/env/file_based/general.yaml | 4 ++-- .../src/commands/chain/genesis.rs | 11 +++++++++- .../src/commands/chain/prepare_en_config.rs | 9 ++++++++- .../src/commands/external_node.rs | 6 +++++- .../zk_inception/src/config_manipulations.rs | 20 ++++++++++++++++--- .../crates/zk_inception/src/defaults.rs | 6 ++++-- 9 files changed, 63 insertions(+), 16 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 6f855d905fd6..c3e981a7f8f2 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -421,6 +421,9 @@ pub(crate) struct OptionalENConfig { #[serde(default = "OptionalENConfig::default_snapshots_recovery_postgres_max_concurrency")] pub snapshots_recovery_postgres_max_concurrency: NonZeroUsize, + #[serde(default)] + pub snapshot_recover_object_store: Option, + /// Enables pruning of the historical node state (Postgres and Merkle tree). The node will retain /// recent state and will continuously remove (prune) old enough parts of the state in the background. #[serde(default)] @@ -614,6 +617,10 @@ impl OptionalENConfig { postgres.max_concurrency, default_snapshots_recovery_postgres_max_concurrency ), + snapshot_recover_object_store: load_config!( + general_config.snapshot_recovery, + object_store + ), pruning_enabled: general_config .pruning .as_ref() @@ -798,9 +805,11 @@ impl OptionalENConfig { } fn from_env() -> anyhow::Result { - envy::prefixed("EN_") + let mut result: OptionalENConfig = envy::prefixed("EN_") .from_env() - .context("could not load external node config") + .context("could not load external node config")?; + result.snapshot_recover_object_store = snapshot_recovery_object_store_config().ok(); + Ok(result) } pub fn polling_interval(&self) -> Duration { @@ -1246,7 +1255,6 @@ impl ExternalNodeConfig<()> { }; let observability = ObservabilityENConfig::from_configs(&general_config)?; let experimental = ExperimentalENConfig::from_configs(&general_config)?; - let api_component = ApiComponentConfig::from_configs(&general_config); let tree_component = TreeComponentConfig::from_configs(&general_config); diff --git a/core/bin/external_node/src/init.rs b/core/bin/external_node/src/init.rs index a9ee796194cc..ddf83a1f5581 100644 --- a/core/bin/external_node/src/init.rs +++ b/core/bin/external_node/src/init.rs @@ -3,6 +3,7 @@ use std::time::Instant; use anyhow::Context as _; +use zksync_config::ObjectStoreConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_health_check::AppHealthCheck; use zksync_node_sync::genesis::perform_genesis_if_needed; @@ -12,12 +13,11 @@ use zksync_snapshots_applier::{SnapshotsApplierConfig, SnapshotsApplierTask}; use zksync_types::{L1BatchNumber, L2ChainId}; use zksync_web3_decl::client::{DynClient, L2}; -use crate::config::snapshot_recovery_object_store_config; - #[derive(Debug)] pub(crate) struct SnapshotRecoveryConfig { /// If not specified, the latest snapshot will be used. pub snapshot_l1_batch_override: Option, + pub object_store_config: Option, } #[derive(Debug)] @@ -90,7 +90,9 @@ pub(crate) async fn ensure_storage_initialized( )?; tracing::warn!("Proceeding with snapshot recovery. This is an experimental feature; use at your own risk"); - let object_store_config = snapshot_recovery_object_store_config()?; + let object_store_config = recovery_config.object_store_config.context( + "Snapshot object store must be presented if snapshot recovery is activated", + )?; let object_store = ObjectStoreFactory::new(object_store_config) .create_store() .await?; diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 8922bd68f3f3..95f6aec9f296 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -965,6 +965,7 @@ async fn run_node( .snapshots_recovery_enabled .then_some(SnapshotRecoveryConfig { snapshot_l1_batch_override: config.experimental.snapshots_recovery_l1_batch, + object_store_config: config.optional.snapshot_recover_object_store.clone(), }); ensure_storage_initialized( connection_pool.clone(), diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index c03c36f28da2..9a671470bc74 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -318,7 +318,7 @@ prometheus: observability: log_format: plain - log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=debug,snapshots_creator=debug" + log_directives: "zksync_node_test_utils=info,zksync_state_keeper=info,zksync_reorg_detector=info,zksync_consistency_checker=info,zksync_metadata_calculator=info,zksync_node_sync=info,zksync_node_consensus=info,zksync_contract_verification_server=info,zksync_node_api_server=info,zksync_tee_verifier_input_producer=info,zksync_node_framework=info,zksync_block_reverter=info,zksync_commitment_generator=info,zksync_node_db_pruner=info,zksync_eth_sender=info,zksync_node_fee_model=info,zksync_node_genesis=info,zksync_house_keeper=info,zksync_proof_data_handler=info,zksync_shared_metrics=info,zksync_node_test_utils=info,zksync_vm_runner=info,zksync_consensus_bft=info,zksync_consensus_network=info,zksync_consensus_storage=info,zksync_core_leftovers=debug,zksync_server=debug,zksync_contract_verifier=debug,zksync_dal=info,zksync_db_connection=info,zksync_eth_client=info,zksync_eth_watch=debug,zksync_storage=info,zksync_db_manager=info,zksync_merkle_tree=info,zksync_state=debug,zksync_utils=debug,zksync_queued_job_processor=info,zksync_types=info,zksync_mempool=debug,loadnext=info,vm=info,zksync_object_store=info,zksync_external_node=info,zksync_witness_generator=info,zksync_prover_fri=info,zksync_witness_vector_generator=info,zksync_web3_decl=debug,zksync_health_check=debug,zksync_proof_fri_compressor=info,vise_exporter=error,snapshots_creator=debug" sentry: url: unset panic_interval: 1800 @@ -334,7 +334,7 @@ protective_reads_writer: first_processed_batch: 0 snapshot_recovery: - enabled: true + enabled: false postgres: max_concurrency: 10 tree: diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 9021cb3edc3d..5f6e5da0fe89 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -13,8 +13,11 @@ use xshell::Shell; use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, - config_manipulations::{update_database_secrets, update_general_config}, + config_manipulations::{ + update_database_secrets, update_general_config, update_rocks_db_config, + }, consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, + defaults::MAIN_ROCKS_DB_PREFIX, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, @@ -49,6 +52,12 @@ pub async fn genesis( shell.create_dir(&config.rocks_db_path)?; update_general_config(shell, config)?; + update_rocks_db_config( + shell, + &config.configs, + &config.rocks_db_path, + MAIN_ROCKS_DB_PREFIX, + )?; update_database_secrets(shell, config, &args.server_db, &args.prover_db)?; logger::note( diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs index 0a69ff0d48a1..451de63d6411 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs @@ -10,6 +10,8 @@ use xshell::Shell; use crate::{ commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal}, + config_manipulations::update_rocks_db_config, + defaults::EN_ROCKS_DB_PREFIX, messages::{ msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, }, @@ -71,9 +73,14 @@ fn prepare_configs( }, other: Default::default(), }; - secrets.save_with_base_path(shell, en_configs_path)?; general_en.save_with_base_path(shell, &en_configs_path)?; + update_rocks_db_config( + shell, + &en_configs_path, + &config.rocks_db_path, + EN_ROCKS_DB_PREFIX, + )?; en_config.save_with_base_path(shell, &en_configs_path)?; Ok(()) diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs index 3fe7476d1c88..7c9aa8b191aa 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs @@ -3,6 +3,7 @@ use common::{ config::global_config, db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, logger, + spinner::Spinner, }; use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; use xshell::Shell; @@ -12,7 +13,8 @@ use crate::{ consts::SERVER_MIGRATIONS, external_node::RunExternalNode, messages::{ - MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_STARTING_SERVER, + MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, + MSG_INITIALIZING_DATABASES_SPINNER, MSG_STARTING_SERVER, }, }; @@ -37,6 +39,7 @@ async fn run_external_node( shell: &Shell, ) -> anyhow::Result<()> { if args.reinit { + let spin = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); let secrets = SecretsConfig::read_with_base_path( shell, chain_config @@ -51,6 +54,7 @@ async fn run_external_node( init_db(&db_config).await?; let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; + spin.finish() } let server = RunExternalNode::new(args.components.clone(), chain_config)?; server.run(shell, args.additional_args.clone()) diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index 9885f4315549..ee0db2bebaa8 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -1,3 +1,5 @@ +use std::path::Path; + use common::db::DatabaseConfig; use config::{ forge_interface::{ @@ -47,11 +49,23 @@ pub(crate) fn update_l1_rpc_url_secret( Ok(()) } +pub(crate) fn update_rocks_db_config( + shell: &Shell, + configs: &Path, + rocks_db_base_path: &Path, + prefix: &str, +) -> anyhow::Result<()> { + let mut general = GeneralConfig::read_with_base_path(shell, &configs)?; + general.db.state_keeper_db_path = + shell.create_dir(rocks_db_base_path.join(&prefix).join(ROCKS_DB_STATE_KEEPER))?; + general.db.merkle_tree.path = + shell.create_dir(rocks_db_base_path.join(&prefix).join(ROCKS_DB_TREE))?; + general.save_with_base_path(shell, &configs)?; + Ok(()) +} + pub(crate) fn update_general_config(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { let mut general = GeneralConfig::read_with_base_path(shell, &config.configs)?; - general.db.state_keeper_db_path = - shell.create_dir(config.rocks_db_path.join(ROCKS_DB_STATE_KEEPER))?; - general.db.merkle_tree.path = shell.create_dir(config.rocks_db_path.join(ROCKS_DB_TREE))?; if config.prover_version != ProverMode::NoProofs { general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); } diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index a0550c5b5efb..40be1293614b 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -9,8 +9,10 @@ lazy_static! { Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); } -pub const ROCKS_DB_STATE_KEEPER: &str = "main/state_keeper"; -pub const ROCKS_DB_TREE: &str = "main/tree"; +pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; +pub const ROCKS_DB_TREE: &str = "tree"; +pub const EN_ROCKS_DB_PREFIX: &str = "en"; +pub const MAIN_ROCKS_DB_PREFIX: &str = "main"; pub const L2_CHAIN_ID: u32 = 271; /// Path to base chain configuration inside zksync-era From 750b4244f7901b9d514fcfe29dedb0881adad06c Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 09:43:34 +0200 Subject: [PATCH 40/56] Fix Signed-off-by: Danil --- zk_toolbox/crates/config/src/chain.rs | 13 +- zk_toolbox/crates/config/src/contracts.rs | 27 ++++- zk_toolbox/crates/config/src/general.rs | 16 +++ zk_toolbox/crates/config/src/genesis.rs | 12 +- zk_toolbox/crates/config/src/secrets.rs | 20 ++++ zk_toolbox/crates/config/src/traits.rs | 13 +- .../zk_inception/src/accept_ownership.rs | 17 ++- .../src/commands/chain/deploy_paymaster.rs | 30 +++-- .../src/commands/chain/genesis.rs | 45 +++---- .../zk_inception/src/commands/chain/init.rs | 68 ++++++----- .../src/commands/chain/initialize_bridges.rs | 16 ++- .../src/commands/chain/prepare_en_config.rs | 24 ++-- .../src/commands/ecosystem/init.rs | 40 ++++--- .../zk_inception/src/config_manipulations.rs | 111 ------------------ zk_toolbox/crates/zk_inception/src/main.rs | 8 +- .../src/{forge_utils.rs => utils/forge.rs} | 0 .../crates/zk_inception/src/utils/mod.rs | 2 + .../crates/zk_inception/src/utils/rocks_db.rs | 32 +++++ 18 files changed, 246 insertions(+), 248 deletions(-) rename zk_toolbox/crates/zk_inception/src/{forge_utils.rs => utils/forge.rs} (100%) create mode 100644 zk_toolbox/crates/zk_inception/src/utils/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs diff --git a/zk_toolbox/crates/config/src/chain.rs b/zk_toolbox/crates/config/src/chain.rs index a2296d9ca166..bfc339f48089 100644 --- a/zk_toolbox/crates/config/src/chain.rs +++ b/zk_toolbox/crates/config/src/chain.rs @@ -4,19 +4,20 @@ use std::{ }; use serde::{Deserialize, Serialize, Serializer}; +use xshell::Shell; + use types::{ BaseToken, ChainId, L1BatchCommitDataGeneratorMode, L1Network, ProverMode, WalletCreation, }; -use xshell::Shell; use crate::{ consts::{ CONFIG_NAME, CONTRACTS_FILE, GENERAL_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, SECRETS_FILE, WALLETS_FILE, }, + ContractsConfig, create_localhost_wallets, - traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, - ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig, + GeneralConfig, GenesisConfig, SecretsConfig, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, WalletsConfig, }; /// Chain configuration file. This file is created in the chain @@ -58,8 +59,8 @@ pub struct ChainConfig { impl Serialize for ChainConfig { fn serialize(&self, serializer: S) -> Result - where - S: Serializer, + where + S: Serializer, { self.get_internal().serialize(serializer) } @@ -107,7 +108,7 @@ impl ChainConfig { config.save(shell, path) } - pub fn save_with_base_path(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + pub fn save_with_base_path(self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { let config = self.get_internal(); config.save_with_base_path(shell, path) } diff --git a/zk_toolbox/crates/config/src/contracts.rs b/zk_toolbox/crates/config/src/contracts.rs index b86b9b0f2958..85cbb3c628f5 100644 --- a/zk_toolbox/crates/config/src/contracts.rs +++ b/zk_toolbox/crates/config/src/contracts.rs @@ -1,11 +1,9 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; -use crate::{ - consts::CONTRACTS_FILE, - forge_interface::deploy_ecosystem::output::DeployL1Output, - traits::{FileConfig, FileConfigWithDefaultName}, -}; +use crate::{consts::CONTRACTS_FILE, forge_interface::deploy_ecosystem::output::DeployL1Output, traits::{FileConfig, FileConfigWithDefaultName}}; +use crate::forge_interface::initialize_bridges::output::InitializeBridgeOutput; +use crate::forge_interface::register_chain::output::RegisterChainOutput; #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct ContractsConfig { @@ -64,6 +62,25 @@ impl ContractsConfig { .diamond_cut_data .clone_from(&deploy_l1_output.contracts_config.diamond_cut_data); } + + pub fn set_chain_contracts( + &mut self, + register_chain_output: &RegisterChainOutput, + ) { + self.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; + self.l1.governance_addr = register_chain_output.governance_addr; + } + + pub fn set_l2_shared_bridge( + &mut self, + initialize_bridges_output: &InitializeBridgeOutput, + ) -> anyhow::Result<()> { + self.bridges.shared.l2_address = + Some(initialize_bridges_output.l2_shared_bridge_proxy); + self.bridges.erc20.l2_address = + Some(initialize_bridges_output.l2_shared_bridge_proxy); + Ok(()) + } } impl FileConfigWithDefaultName for ContractsConfig { diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 1fcc22801474..6d77e4d7daa6 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -4,6 +4,11 @@ use serde::{Deserialize, Serialize}; use crate::{consts::GENERAL_FILE, traits::FileConfigWithDefaultName}; +pub struct RocksDbs { + pub state_keeper: PathBuf, + pub merkle_tree: PathBuf, +} + #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GeneralConfig { pub db: RocksDBConfig, @@ -13,6 +18,17 @@ pub struct GeneralConfig { pub other: serde_json::Value, } +impl GeneralConfig { + pub fn set_rocks_db_config( + &mut self, + rocks_dbs: RocksDbs, + ) -> anyhow::Result<()> { + self.db.state_keeper_db_path = rocks_dbs.state_keeper; + self.db.merkle_tree.path = rocks_dbs.merkle_tree; + Ok(()) + } +} + impl FileConfigWithDefaultName for GeneralConfig { const FILE_NAME: &'static str = GENERAL_FILE; } diff --git a/zk_toolbox/crates/config/src/genesis.rs b/zk_toolbox/crates/config/src/genesis.rs index 4e3d931ea0f0..2d0d3b628b2d 100644 --- a/zk_toolbox/crates/config/src/genesis.rs +++ b/zk_toolbox/crates/config/src/genesis.rs @@ -1,8 +1,9 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; + use types::{ChainId, L1BatchCommitDataGeneratorMode, ProtocolSemanticVersion}; -use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName}; +use crate::{ChainConfig, consts::GENESIS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GenesisConfig { @@ -21,6 +22,15 @@ pub struct GenesisConfig { pub other: serde_json::Value, } +impl GenesisConfig { + pub fn set_from_config(&mut self, config: &ChainConfig) { + self.l2_chain_id = config.chain_id; + self.l1_chain_id = config.l1_network.chain_id(); + self.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); + } +} + + impl FileConfigWithDefaultName for GenesisConfig { const FILE_NAME: &'static str = GENESIS_FILE; } diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index 86d0585e354d..ba79d28f6c17 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -1,6 +1,8 @@ use serde::{Deserialize, Serialize}; use url::Url; +use common::db::DatabaseConfig; + use crate::{consts::SECRETS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -27,6 +29,24 @@ pub struct SecretsConfig { pub other: serde_json::Value, } +impl SecretsConfig { + pub fn set_databases(&mut self, + server_db_config: &DatabaseConfig, + prover_db_config: &DatabaseConfig, + ) { + self.database.server_url = server_db_config.full_url(); + self.database.prover_url = Some(prover_db_config.full_url()); + } + + pub fn set_l1_rpc_url( + &mut self, + l1_rpc_url: String, + ) { + self.l1.l1_rpc_url = l1_rpc_url; + } +} + + impl FileConfigWithDefaultName for SecretsConfig { const FILE_NAME: &'static str = SECRETS_FILE; } diff --git a/zk_toolbox/crates/config/src/traits.rs b/zk_toolbox/crates/config/src/traits.rs index 85c73e99f99b..098e239c3155 100644 --- a/zk_toolbox/crates/config/src/traits.rs +++ b/zk_toolbox/crates/config/src/traits.rs @@ -1,11 +1,12 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context}; +use serde::{de::DeserializeOwned, Serialize}; +use xshell::Shell; + use common::files::{ read_json_file, read_toml_file, read_yaml_file, save_json_file, save_toml_file, save_yaml_file, }; -use serde::{de::DeserializeOwned, Serialize}; -use xshell::Shell; pub trait FileConfig {} @@ -18,11 +19,17 @@ pub trait FileConfigWithDefaultName { } impl FileConfig for T where T: FileConfigWithDefaultName {} + impl ReadConfig for T where T: FileConfig + Clone + DeserializeOwned {} + impl SaveConfig for T where T: FileConfig + Serialize {} + impl SaveConfigWithComment for T where T: FileConfig + Serialize {} + impl ReadConfigWithBasePath for T where T: FileConfigWithDefaultName + Clone + DeserializeOwned {} + impl SaveConfigWithBasePath for T where T: FileConfigWithDefaultName + Serialize {} + impl SaveConfigWithCommentAndBasePath for T where T: FileConfigWithDefaultName + Serialize {} /// Reads a config file from a given path, correctly parsing file extension. @@ -98,7 +105,7 @@ pub trait SaveConfigWithComment: Serialize + Sized { /// Saves a config file from a base path, correctly parsing file extension. /// Supported file extensions are: `yaml`, `yml`, `toml`. pub trait SaveConfigWithCommentAndBasePath: - SaveConfigWithComment + FileConfigWithDefaultName +SaveConfigWithComment + FileConfigWithDefaultName { fn save_with_comment_and_base_path( &self, diff --git a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs index 830da513d4f0..cb4ed373a1d0 100644 --- a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs +++ b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs @@ -1,21 +1,20 @@ +use ethers::types::{Address, H256}; +use xshell::Shell; + use common::{ forge::{Forge, ForgeScript, ForgeScriptArgs}, spinner::Spinner, }; use config::{ + EcosystemConfig, forge_interface::{ accept_ownership::AcceptOwnershipInput, script_params::ACCEPT_GOVERNANCE_SCRIPT_PARAMS, }, traits::SaveConfig, - EcosystemConfig, }; -use ethers::types::{Address, H256}; -use xshell::Shell; -use crate::{ - forge_utils::{check_the_balance, fill_forge_private_key}, - messages::MSG_ACCEPTING_GOVERNANCE_SPINNER, -}; +use crate::messages::MSG_ACCEPTING_GOVERNANCE_SPINNER; +use crate::utils::forge::{check_the_balance, fill_forge_private_key}; pub async fn accept_admin( shell: &Shell, @@ -44,7 +43,7 @@ pub async fn accept_admin( target_address, forge, ) - .await + .await } pub async fn accept_owner( @@ -74,7 +73,7 @@ pub async fn accept_owner( target_address, forge, ) - .await + .await } async fn accept_ownership( diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs index fe8dcdc562b2..c1667383c957 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs @@ -1,24 +1,19 @@ use anyhow::Context; +use xshell::Shell; + use common::{ config::global_config, forge::{Forge, ForgeScriptArgs}, spinner::Spinner, }; -use config::{ - forge_interface::{ - paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, - script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, - }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, -}; -use xshell::Shell; +use config::{ChainConfig, ContractsConfig, EcosystemConfig, forge_interface::{ + paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, + script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, +}, traits::{ReadConfig, SaveConfig}}; +use config::traits::SaveConfigWithBasePath; -use crate::{ - config_manipulations::update_paymaster, - forge_utils::{check_the_balance, fill_forge_private_key}, - messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}, -}; +use crate::messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}; +use crate::utils::forge::{check_the_balance, fill_forge_private_key}; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); @@ -26,12 +21,15 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_config = ecosystem_config .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; - deploy_paymaster(shell, &chain_config, args).await + let mut contracts = chain_config.get_contracts_config()?; + deploy_paymaster(shell, &chain_config, &mut contracts, args).await?; + contracts.save_with_base_path(shell, chain_config.configs) } pub async fn deploy_paymaster( shell: &Shell, chain_config: &ChainConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { let input = DeployPaymasterInput::new(chain_config)?; @@ -63,6 +61,6 @@ pub async fn deploy_paymaster( DEPLOY_PAYMASTER_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_paymaster(shell, chain_config, &output)?; + contracts_config.l2.testnet_paymaster_addr = output.paymaster; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 5f6e5da0fe89..2d1e6fddfc67 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -1,21 +1,20 @@ use std::path::PathBuf; use anyhow::Context; +use xshell::Shell; + use common::{ config::global_config, - db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, + db::{DatabaseConfig, drop_db_if_exists, init_db, migrate_db}, logger, spinner::Spinner, }; -use config::{ChainConfig, EcosystemConfig}; -use xshell::Shell; +use config::{ChainConfig, EcosystemConfig, SecretsConfig}; +use config::traits::{FileConfigWithDefaultName, ReadConfigWithBasePath, SaveConfigWithBasePath}; +use types::ProverMode; -use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, - config_manipulations::{ - update_database_secrets, update_general_config, update_rocks_db_config, - }, consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, defaults::MAIN_ROCKS_DB_PREFIX, messages::{ @@ -26,8 +25,11 @@ use crate::{ MSG_STARTING_GENESIS_SPINNER, }, server::{RunServer, ServerMode}, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; +use super::args::genesis::GenesisArgsFinal; + pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -47,18 +49,21 @@ pub async fn genesis( shell: &Shell, config: &ChainConfig, ) -> anyhow::Result<()> { - // Clean the rocksdb - shell.remove_path(&config.rocks_db_path)?; shell.create_dir(&config.rocks_db_path)?; - update_general_config(shell, config)?; - update_rocks_db_config( - shell, - &config.configs, - &config.rocks_db_path, - MAIN_ROCKS_DB_PREFIX, - )?; - update_database_secrets(shell, config, &args.server_db, &args.prover_db)?; + let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main).context("Failed to create rocks db path")?; + let mut general = config.get_general_config()?; + general.set_rocks_db_config(rocks_db)?; + if config.prover_version != ProverMode::NoProofs { + general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); + } + general.save_with_base_path(shell, &config.configs)?; + + let mut secrets = config.get_secrets_config()?; + secrets.set_databases( + &args.server_db, &args.prover_db, + ); + secrets.save_with_base_path(&shell, &config.configs)?; logger::note( MSG_SELECTED_CONFIG, @@ -78,7 +83,7 @@ pub async fn genesis( config.link_to_code.clone(), args.dont_drop, ) - .await?; + .await?; spinner.finish(); let spinner = Spinner::new(MSG_STARTING_GENESIS_SPINNER); @@ -111,7 +116,7 @@ async fn initialize_databases( path_to_server_migration, &server_db_config.full_url(), ) - .await?; + .await?; if global_config().verbose { logger::debug(MSG_INITIALIZING_PROVER_DATABASE) @@ -128,7 +133,7 @@ async fn initialize_databases( path_to_prover_migration, &prover_db_config.full_url(), ) - .await?; + .await?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index bb7b60e16738..dd370b424883 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -1,4 +1,6 @@ use anyhow::Context; +use xshell::{cmd, Shell}; + use common::{ cmd::Cmd, config::global_config, @@ -6,31 +8,26 @@ use common::{ logger, spinner::Spinner, }; -use config::{ - copy_configs, - forge_interface::{ - register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, - script_params::REGISTER_CHAIN_SCRIPT_PARAMS, - }, - traits::{ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath}, - ChainConfig, ContractsConfig, EcosystemConfig, -}; -use xshell::{cmd, Shell}; +use config::{ChainConfig, ContractsConfig, copy_configs, EcosystemConfig, forge_interface::{ + register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, + script_params::REGISTER_CHAIN_SCRIPT_PARAMS, +}, traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}}; -use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ - args::init::InitArgs, deploy_paymaster, genesis::genesis, initialize_bridges, + args::init::InitArgs, deploy_paymaster, initialize_bridges, }, - config_manipulations::{update_genesis, update_l1_contracts, update_l1_rpc_url_secret}, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ - msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, - MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR, - MSG_GENESIS_DATABASE_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, + MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_INITIALIZED, + MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, + msg_initializing_chain, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, }, }; +use crate::commands::chain::genesis::genesis; +use crate::utils::forge::{check_the_balance, fill_forge_private_key}; + +use super::args::init::InitArgsFinal; pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); @@ -57,23 +54,27 @@ pub async fn init( ) -> anyhow::Result<()> { copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; build_l1_contracts(shell, ecosystem_config)?; - update_genesis(shell, chain_config)?; - update_l1_rpc_url_secret(shell, chain_config, init_args.l1_rpc_url.clone())?; - let mut contracts_config = - ContractsConfig::read_with_base_path(shell, &ecosystem_config.config)?; - contracts_config.l1.base_token_addr = chain_config.base_token.address; + + let mut genesis_config = chain_config.get_genesis_config()?; + genesis_config.set_from_config(&chain_config); + genesis_config.save_with_base_path(shell, &chain_config.configs)?; + // Copy ecosystem contracts + let mut contracts_config = chain_config.get_contracts_config()?; + contracts_config.l1.base_token_addr = chain_config.base_token.address; contracts_config.save_with_base_path(shell, &chain_config.configs)?; let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); - contracts_config = register_chain( + register_chain( shell, init_args.forge_args.clone(), ecosystem_config, chain_config, + &mut contracts_config, init_args.l1_rpc_url.clone(), ) - .await?; + .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); accept_admin( @@ -85,7 +86,7 @@ pub async fn init( &init_args.forge_args.clone(), init_args.l1_rpc_url.clone(), ) - .await?; + .await?; spinner.finish(); initialize_bridges::initialize_bridges( @@ -94,13 +95,17 @@ pub async fn init( ecosystem_config, init_args.forge_args.clone(), ) - .await?; + .await?; if init_args.deploy_paymaster { - deploy_paymaster::deploy_paymaster(shell, chain_config, init_args.forge_args.clone()) + deploy_paymaster::deploy_paymaster(shell, chain_config, &mut contracts_config, init_args.forge_args.clone()) .await?; } + let mut secrets = chain_config.get_secrets_config()?; + secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); + secrets.save_with_base_path(shell, &chain_config.configs)?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; genesis(init_args.genesis_args.clone(), shell, chain_config) .await .context(MSG_GENESIS_DATABASE_ERR)?; @@ -113,13 +118,11 @@ async fn register_chain( forge_args: ForgeScriptArgs, config: &EcosystemConfig, chain_config: &ChainConfig, + contracts: &mut ContractsConfig, l1_rpc_url: String, -) -> anyhow::Result { +) -> anyhow::Result<()> { let deploy_config_path = REGISTER_CHAIN_SCRIPT_PARAMS.input(&config.link_to_code); - let contracts = config - .get_contracts_config() - .context(MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR)?; let deploy_config = RegisterChainL1Config::new(chain_config, &contracts)?; deploy_config.save(shell, deploy_config_path)?; @@ -137,7 +140,8 @@ async fn register_chain( shell, REGISTER_CHAIN_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l1_contracts(shell, chain_config, ®ister_chain_output) + contracts.set_chain_contracts(®ister_chain_output); + Ok(()) } fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs index 4a81a2b26f1b..1edba5387a6b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs @@ -1,6 +1,8 @@ use std::path::Path; use anyhow::Context; +use xshell::{cmd, Shell}; + use common::{ cmd::Cmd, config::global_config, @@ -8,20 +10,16 @@ use common::{ spinner::Spinner, }; use config::{ + ChainConfig, + EcosystemConfig, forge_interface::{ initialize_bridges::{input::InitializeBridgeInput, output::InitializeBridgeOutput}, script_params::INITIALIZE_BRIDGES_SCRIPT_PARAMS, - }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + }, traits::{ReadConfig, SaveConfig}, }; -use xshell::{cmd, Shell}; -use crate::{ - config_manipulations::update_l2_shared_bridge, - forge_utils::{check_the_balance, fill_forge_private_key}, - messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}, -}; +use crate::messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}; +use crate::utils::forge::{check_the_balance, fill_forge_private_key}; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs index 451de63d6411..65e87010e84e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs @@ -1,21 +1,22 @@ use std::path::Path; use anyhow::Context; +use xshell::Shell; + use common::{config::global_config, logger}; use config::{ - external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, DatabaseSecrets, - EcosystemConfig, L1Secret, SecretsConfig, + ChainConfig, DatabaseSecrets, EcosystemConfig, external_node::ENConfig, + L1Secret, SecretsConfig, traits::SaveConfigWithBasePath, }; -use xshell::Shell; use crate::{ - commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal}, - config_manipulations::update_rocks_db_config, - defaults::EN_ROCKS_DB_PREFIX, + commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal} + , messages::{ - msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, + MSG_CHAIN_NOT_INITIALIZED, msg_preparing_en_config_is_done, MSG_PREPARING_EN_CONFIGS, }, }; +use crate::utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}; pub fn run(shell: &Shell, args: PrepareConfigArgs) -> anyhow::Result<()> { logger::info(MSG_PREPARING_EN_CONFIGS); @@ -74,13 +75,10 @@ fn prepare_configs( other: Default::default(), }; secrets.save_with_base_path(shell, en_configs_path)?; + let dirs = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::ExternalNode)?; + general_en.set_rocks_db_config(dirs)?; + general_en.save_with_base_path(shell, &en_configs_path)?; - update_rocks_db_config( - shell, - &en_configs_path, - &config.rocks_db_path, - EN_ROCKS_DB_PREFIX, - )?; en_config.save_with_base_path(shell, &en_configs_path)?; Ok(()) diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs index fecda40c7760..49e8687eb809 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs @@ -4,16 +4,20 @@ use std::{ }; use anyhow::Context; +use xshell::{cmd, Shell}; + use common::{ cmd::Cmd, config::global_config, forge::{Forge, ForgeScriptArgs}, logger, - spinner::Spinner, Prompt, + spinner::Spinner, }; use config::{ - forge_interface::{ + ChainConfig, + ContractsConfig, + EcosystemConfig, forge_interface::{ deploy_ecosystem::{ input::{ DeployErc20Config, DeployL1Config, Erc20DeploymentConfig, InitialDeploymentConfig, @@ -21,17 +25,13 @@ use config::{ output::{DeployErc20Output, DeployL1Output}, }, script_params::{DEPLOY_ECOSYSTEM_SCRIPT_PARAMS, DEPLOY_ERC20_SCRIPT_PARAMS}, - }, - traits::{ + }, GenesisConfig, traits::{ FileConfigWithDefaultName, ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath, }, - ChainConfig, ContractsConfig, EcosystemConfig, GenesisConfig, }; use types::{L1Network, ProverMode, WalletCreation}; -use xshell::{cmd, Shell}; -use super::args::init::{EcosystemArgsFinal, EcosystemInitArgs, EcosystemInitArgsFinal}; use crate::{ accept_ownership::accept_owner, commands::{ @@ -41,15 +41,17 @@ use crate::{ }, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, - forge_utils::{check_the_balance, fill_forge_private_key}, messages::{ - msg_ecosystem_initialized, msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED, - MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, + MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, MSG_DEPLOYING_ERC20_SPINNER, MSG_DISTRIBUTING_ETH_SPINNER, MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, + msg_ecosystem_initialized, msg_initializing_chain, MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER, }, }; +use crate::utils::forge::{check_the_balance, fill_forge_private_key}; + +use super::args::init::{EcosystemArgsFinal, EcosystemInitArgs, EcosystemInitArgsFinal}; pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -70,7 +72,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &ecosystem_config, &initial_deployment_config, ) - .await?; + .await?; if final_ecosystem_args.deploy_erc20 { logger::info(MSG_DEPLOYING_ERC20); @@ -86,7 +88,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { final_ecosystem_args.forge_args.clone(), final_ecosystem_args.ecosystem.l1_rpc_url.clone(), ) - .await?; + .await?; } // If the name of chain passed then we deploy exactly this chain otherwise deploy all chains @@ -114,7 +116,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &chain_config, final_ecosystem_args.ecosystem.l1_rpc_url.clone(), ) - .await?; + .await?; chain::init::init( &mut chain_init_args, @@ -122,7 +124,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &ecosystem_config, &chain_config, ) - .await?; + .await?; } logger::outro(msg_ecosystem_initialized(&list_of_chains.join(","))); @@ -157,7 +159,7 @@ pub async fn distribute_eth( ecosystem_config.l1_network.chain_id(), AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, ) - .await?; + .await?; spinner.finish(); } Ok(()) @@ -181,7 +183,7 @@ async fn init( ecosystem_config, initial_deployment_config, ) - .await?; + .await?; contracts.save_with_base_path(shell, &ecosystem_config.config)?; Ok(contracts) } @@ -237,7 +239,7 @@ async fn deploy_ecosystem( initial_deployment_config, ecosystem.l1_rpc_url.clone(), ) - .await; + .await; } let ecosystem_contracts_path = match &ecosystem.ecosystem_contracts_path { @@ -332,7 +334,7 @@ async fn deploy_ecosystem_inner( &forge_args, l1_rpc_url.clone(), ) - .await?; + .await?; accept_owner( shell, @@ -343,7 +345,7 @@ async fn deploy_ecosystem_inner( &forge_args, l1_rpc_url.clone(), ) - .await?; + .await?; Ok(contracts_config) } diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index ee0db2bebaa8..e69de29bb2d1 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -1,111 +0,0 @@ -use std::path::Path; - -use common::db::DatabaseConfig; -use config::{ - forge_interface::{ - initialize_bridges::output::InitializeBridgeOutput, paymaster::DeployPaymasterOutput, - register_chain::output::RegisterChainOutput, - }, - traits::{ReadConfigWithBasePath, SaveConfigWithBasePath}, - ChainConfig, ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, -}; -use types::ProverMode; -use xshell::Shell; - -use crate::defaults::{ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE}; - -pub(crate) fn update_genesis(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut genesis = GenesisConfig::read_with_base_path(shell, &config.configs)?; - - genesis.l2_chain_id = config.chain_id; - genesis.l1_chain_id = config.l1_network.chain_id(); - genesis.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); - - genesis.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_database_secrets( - shell: &Shell, - config: &ChainConfig, - server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.database.server_url = server_db_config.full_url(); - secrets.database.prover_url = Some(prover_db_config.full_url()); - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_l1_rpc_url_secret( - shell: &Shell, - config: &ChainConfig, - l1_rpc_url: String, -) -> anyhow::Result<()> { - let mut secrets = SecretsConfig::read_with_base_path(shell, &config.configs)?; - secrets.l1.l1_rpc_url = l1_rpc_url; - secrets.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub(crate) fn update_rocks_db_config( - shell: &Shell, - configs: &Path, - rocks_db_base_path: &Path, - prefix: &str, -) -> anyhow::Result<()> { - let mut general = GeneralConfig::read_with_base_path(shell, &configs)?; - general.db.state_keeper_db_path = - shell.create_dir(rocks_db_base_path.join(&prefix).join(ROCKS_DB_STATE_KEEPER))?; - general.db.merkle_tree.path = - shell.create_dir(rocks_db_base_path.join(&prefix).join(ROCKS_DB_TREE))?; - general.save_with_base_path(shell, &configs)?; - Ok(()) -} - -pub(crate) fn update_general_config(shell: &Shell, config: &ChainConfig) -> anyhow::Result<()> { - let mut general = GeneralConfig::read_with_base_path(shell, &config.configs)?; - if config.prover_version != ProverMode::NoProofs { - general.eth.sender.proof_sending_mode = "ONLY_REAL_PROOFS".to_string(); - } - general.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_l1_contracts( - shell: &Shell, - config: &ChainConfig, - register_chain_output: &RegisterChainOutput, -) -> anyhow::Result { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; - contracts_config.l1.governance_addr = register_chain_output.governance_addr; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(contracts_config) -} - -pub fn update_l2_shared_bridge( - shell: &Shell, - config: &ChainConfig, - initialize_bridges_output: &InitializeBridgeOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.bridges.shared.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.bridges.erc20.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} - -pub fn update_paymaster( - shell: &Shell, - config: &ChainConfig, - paymaster_output: &DeployPaymasterOutput, -) -> anyhow::Result<()> { - let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; - contracts_config.l2.testnet_paymaster_addr = paymaster_output.paymaster; - contracts_config.save_with_base_path(shell, &config.configs)?; - Ok(()) -} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index d102e0d69a95..32b7f603ee26 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -1,11 +1,12 @@ use clap::{command, Parser, Subcommand}; +use xshell::Shell; + use common::{ check_prerequisites, - config::{global_config, init_global_config, GlobalConfig}, + config::{global_config, GlobalConfig, init_global_config}, init_prompt_theme, logger, }; use config::EcosystemConfig; -use xshell::Shell; use crate::commands::{ args::{RunExternalNodeArgs, RunServerArgs}, @@ -15,13 +16,12 @@ use crate::commands::{ pub mod accept_ownership; mod commands; -mod config_manipulations; mod consts; mod defaults; pub mod external_node; -pub mod forge_utils; mod messages; pub mod server; +mod utils; #[derive(Parser, Debug)] #[command(version, about)] diff --git a/zk_toolbox/crates/zk_inception/src/forge_utils.rs b/zk_toolbox/crates/zk_inception/src/utils/forge.rs similarity index 100% rename from zk_toolbox/crates/zk_inception/src/forge_utils.rs rename to zk_toolbox/crates/zk_inception/src/utils/forge.rs diff --git a/zk_toolbox/crates/zk_inception/src/utils/mod.rs b/zk_toolbox/crates/zk_inception/src/utils/mod.rs new file mode 100644 index 000000000000..a84f0a336de5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod forge; +pub mod rocks_db; diff --git a/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs new file mode 100644 index 000000000000..17b751790cb7 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs @@ -0,0 +1,32 @@ +use std::path::Path; + +use xshell::Shell; + +use config::RocksDbs; + +use crate::defaults::{EN_ROCKS_DB_PREFIX, MAIN_ROCKS_DB_PREFIX, ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE}; + +pub enum RocksDBDirOption { + Main, + ExternalNode, +} + +impl RocksDBDirOption { + pub fn prefix(&self) -> &str { + match self { + RocksDBDirOption::Main => MAIN_ROCKS_DB_PREFIX, + RocksDBDirOption::ExternalNode => EN_ROCKS_DB_PREFIX + } + } +} + +pub fn recreate_rocksdb_dirs(shell: &Shell, rocks_db_path: &Path, option: RocksDBDirOption) -> anyhow::Result { + let state_keeper = rocks_db_path.join(option.prefix()).join(ROCKS_DB_STATE_KEEPER); + shell.remove_path(&state_keeper)?; + let merkle_tree = rocks_db_path.join(option.prefix()).join(ROCKS_DB_TREE); + shell.remove_path(&merkle_tree)?; + Ok(RocksDbs { + state_keeper: shell.create_dir(state_keeper)?, + merkle_tree: shell.create_dir(merkle_tree)?, + }) +} From f2e32152bf0bdd1c50eb5989bfb89aed3711f5f5 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 12:06:34 +0200 Subject: [PATCH 41/56] Update working with configs Signed-off-by: Danil --- etc/env/file_based/contracts.yaml | 4 ++ zk_toolbox/crates/config/src/chain.rs | 11 ++-- zk_toolbox/crates/config/src/contracts.rs | 23 ++++---- zk_toolbox/crates/config/src/general.rs | 5 +- zk_toolbox/crates/config/src/genesis.rs | 6 +-- zk_toolbox/crates/config/src/secrets.rs | 16 +++--- zk_toolbox/crates/config/src/traits.rs | 7 ++- .../zk_inception/src/accept_ownership.rs | 17 +++--- .../src/commands/chain/deploy_paymaster.rs | 22 ++++---- .../src/commands/chain/genesis.rs | 25 ++++----- .../zk_inception/src/commands/chain/init.rs | 53 +++++++++++-------- .../src/commands/chain/initialize_bridges.rs | 29 ++++++---- .../src/commands/chain/prepare_en_config.rs | 14 +++-- .../src/commands/ecosystem/init.rs | 40 +++++++------- zk_toolbox/crates/zk_inception/src/main.rs | 5 +- .../crates/zk_inception/src/utils/rocks_db.rs | 19 ++++--- 16 files changed, 156 insertions(+), 140 deletions(-) diff --git a/etc/env/file_based/contracts.yaml b/etc/env/file_based/contracts.yaml index e6f175a2727a..4777cc56ab8f 100644 --- a/etc/env/file_based/contracts.yaml +++ b/etc/env/file_based/contracts.yaml @@ -1,3 +1,5 @@ +create2_factory_addr: 0x0000000000000000000000000000000000000000 +create2_factory_salt: 0x7a084e42e7c4851b6efbe654f7955cca0f1e3a09ed32d56c569ca16044b52a3c l1: diamond_proxy_addr: "0x0000000000000000000000000000000000000000" default_upgrade_addr: "0x0000000000000000000000000000000000000000" @@ -18,3 +20,5 @@ ecosystem_contracts: state_transition_proxy_addr: "0x0000000000000000000000000000000000000000" bridgehub_proxy_addr: "0x0000000000000000000000000000000000000000" transparent_proxy_admin_addr: "0x0000000000000000000000000000000000000000" + validator_timelock_addr: "0x0000000000000000000000000000000000000000" + diamond_cut_data: "0x" diff --git a/zk_toolbox/crates/config/src/chain.rs b/zk_toolbox/crates/config/src/chain.rs index bfc339f48089..f00bee175c0d 100644 --- a/zk_toolbox/crates/config/src/chain.rs +++ b/zk_toolbox/crates/config/src/chain.rs @@ -4,20 +4,19 @@ use std::{ }; use serde::{Deserialize, Serialize, Serializer}; -use xshell::Shell; - use types::{ BaseToken, ChainId, L1BatchCommitDataGeneratorMode, L1Network, ProverMode, WalletCreation, }; +use xshell::Shell; use crate::{ consts::{ CONFIG_NAME, CONTRACTS_FILE, GENERAL_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, SECRETS_FILE, WALLETS_FILE, }, - ContractsConfig, create_localhost_wallets, - GeneralConfig, GenesisConfig, SecretsConfig, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, WalletsConfig, + traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ContractsConfig, GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig, }; /// Chain configuration file. This file is created in the chain @@ -59,8 +58,8 @@ pub struct ChainConfig { impl Serialize for ChainConfig { fn serialize(&self, serializer: S) -> Result - where - S: Serializer, + where + S: Serializer, { self.get_internal().serialize(serializer) } diff --git a/zk_toolbox/crates/config/src/contracts.rs b/zk_toolbox/crates/config/src/contracts.rs index 85cbb3c628f5..a847c8a4cc93 100644 --- a/zk_toolbox/crates/config/src/contracts.rs +++ b/zk_toolbox/crates/config/src/contracts.rs @@ -1,9 +1,15 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; -use crate::{consts::CONTRACTS_FILE, forge_interface::deploy_ecosystem::output::DeployL1Output, traits::{FileConfig, FileConfigWithDefaultName}}; -use crate::forge_interface::initialize_bridges::output::InitializeBridgeOutput; -use crate::forge_interface::register_chain::output::RegisterChainOutput; +use crate::{ + consts::CONTRACTS_FILE, + forge_interface::{ + deploy_ecosystem::output::DeployL1Output, + initialize_bridges::output::InitializeBridgeOutput, + register_chain::output::RegisterChainOutput, + }, + traits::{FileConfig, FileConfigWithDefaultName}, +}; #[derive(Debug, Deserialize, Serialize, Clone, Default)] pub struct ContractsConfig { @@ -63,10 +69,7 @@ impl ContractsConfig { .clone_from(&deploy_l1_output.contracts_config.diamond_cut_data); } - pub fn set_chain_contracts( - &mut self, - register_chain_output: &RegisterChainOutput, - ) { + pub fn set_chain_contracts(&mut self, register_chain_output: &RegisterChainOutput) { self.l1.diamond_proxy_addr = register_chain_output.diamond_proxy_addr; self.l1.governance_addr = register_chain_output.governance_addr; } @@ -75,10 +78,8 @@ impl ContractsConfig { &mut self, initialize_bridges_output: &InitializeBridgeOutput, ) -> anyhow::Result<()> { - self.bridges.shared.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); - self.bridges.erc20.l2_address = - Some(initialize_bridges_output.l2_shared_bridge_proxy); + self.bridges.shared.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + self.bridges.erc20.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); Ok(()) } } diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 6d77e4d7daa6..74575f64e8c6 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -19,10 +19,7 @@ pub struct GeneralConfig { } impl GeneralConfig { - pub fn set_rocks_db_config( - &mut self, - rocks_dbs: RocksDbs, - ) -> anyhow::Result<()> { + pub fn set_rocks_db_config(&mut self, rocks_dbs: RocksDbs) -> anyhow::Result<()> { self.db.state_keeper_db_path = rocks_dbs.state_keeper; self.db.merkle_tree.path = rocks_dbs.merkle_tree; Ok(()) diff --git a/zk_toolbox/crates/config/src/genesis.rs b/zk_toolbox/crates/config/src/genesis.rs index 2d0d3b628b2d..a6dbb3dcc955 100644 --- a/zk_toolbox/crates/config/src/genesis.rs +++ b/zk_toolbox/crates/config/src/genesis.rs @@ -1,9 +1,8 @@ use ethers::types::{Address, H256}; use serde::{Deserialize, Serialize}; - use types::{ChainId, L1BatchCommitDataGeneratorMode, ProtocolSemanticVersion}; -use crate::{ChainConfig, consts::GENESIS_FILE, traits::FileConfigWithDefaultName}; +use crate::{consts::GENESIS_FILE, traits::FileConfigWithDefaultName, ChainConfig}; #[derive(Debug, Deserialize, Serialize, Clone)] pub struct GenesisConfig { @@ -23,14 +22,13 @@ pub struct GenesisConfig { } impl GenesisConfig { - pub fn set_from_config(&mut self, config: &ChainConfig) { + pub fn set_from_chain_config(&mut self, config: &ChainConfig) { self.l2_chain_id = config.chain_id; self.l1_chain_id = config.l1_network.chain_id(); self.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); } } - impl FileConfigWithDefaultName for GenesisConfig { const FILE_NAME: &'static str = GENESIS_FILE; } diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index ba79d28f6c17..98a9be6ffe61 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -1,8 +1,7 @@ +use common::db::DatabaseConfig; use serde::{Deserialize, Serialize}; use url::Url; -use common::db::DatabaseConfig; - use crate::{consts::SECRETS_FILE, traits::FileConfigWithDefaultName}; #[derive(Debug, Clone, Serialize, Deserialize)] @@ -30,23 +29,20 @@ pub struct SecretsConfig { } impl SecretsConfig { - pub fn set_databases(&mut self, - server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, + pub fn set_databases( + &mut self, + server_db_config: &DatabaseConfig, + prover_db_config: &DatabaseConfig, ) { self.database.server_url = server_db_config.full_url(); self.database.prover_url = Some(prover_db_config.full_url()); } - pub fn set_l1_rpc_url( - &mut self, - l1_rpc_url: String, - ) { + pub fn set_l1_rpc_url(&mut self, l1_rpc_url: String) { self.l1.l1_rpc_url = l1_rpc_url; } } - impl FileConfigWithDefaultName for SecretsConfig { const FILE_NAME: &'static str = SECRETS_FILE; } diff --git a/zk_toolbox/crates/config/src/traits.rs b/zk_toolbox/crates/config/src/traits.rs index 098e239c3155..79ae3a187a8b 100644 --- a/zk_toolbox/crates/config/src/traits.rs +++ b/zk_toolbox/crates/config/src/traits.rs @@ -1,12 +1,11 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context}; -use serde::{de::DeserializeOwned, Serialize}; -use xshell::Shell; - use common::files::{ read_json_file, read_toml_file, read_yaml_file, save_json_file, save_toml_file, save_yaml_file, }; +use serde::{de::DeserializeOwned, Serialize}; +use xshell::Shell; pub trait FileConfig {} @@ -105,7 +104,7 @@ pub trait SaveConfigWithComment: Serialize + Sized { /// Saves a config file from a base path, correctly parsing file extension. /// Supported file extensions are: `yaml`, `yml`, `toml`. pub trait SaveConfigWithCommentAndBasePath: -SaveConfigWithComment + FileConfigWithDefaultName + SaveConfigWithComment + FileConfigWithDefaultName { fn save_with_comment_and_base_path( &self, diff --git a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs index cb4ed373a1d0..179cb696ac3d 100644 --- a/zk_toolbox/crates/zk_inception/src/accept_ownership.rs +++ b/zk_toolbox/crates/zk_inception/src/accept_ownership.rs @@ -1,20 +1,21 @@ -use ethers::types::{Address, H256}; -use xshell::Shell; - use common::{ forge::{Forge, ForgeScript, ForgeScriptArgs}, spinner::Spinner, }; use config::{ - EcosystemConfig, forge_interface::{ accept_ownership::AcceptOwnershipInput, script_params::ACCEPT_GOVERNANCE_SCRIPT_PARAMS, }, traits::SaveConfig, + EcosystemConfig, }; +use ethers::types::{Address, H256}; +use xshell::Shell; -use crate::messages::MSG_ACCEPTING_GOVERNANCE_SPINNER; -use crate::utils::forge::{check_the_balance, fill_forge_private_key}; +use crate::{ + messages::MSG_ACCEPTING_GOVERNANCE_SPINNER, + utils::forge::{check_the_balance, fill_forge_private_key}, +}; pub async fn accept_admin( shell: &Shell, @@ -43,7 +44,7 @@ pub async fn accept_admin( target_address, forge, ) - .await + .await } pub async fn accept_owner( @@ -73,7 +74,7 @@ pub async fn accept_owner( target_address, forge, ) - .await + .await } async fn accept_ownership( diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs index c1667383c957..4f82a92c2edc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/deploy_paymaster.rs @@ -1,19 +1,23 @@ use anyhow::Context; -use xshell::Shell; - use common::{ config::global_config, forge::{Forge, ForgeScriptArgs}, spinner::Spinner, }; -use config::{ChainConfig, ContractsConfig, EcosystemConfig, forge_interface::{ - paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, - script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, -}, traits::{ReadConfig, SaveConfig}}; -use config::traits::SaveConfigWithBasePath; +use config::{ + forge_interface::{ + paymaster::{DeployPaymasterInput, DeployPaymasterOutput}, + script_params::DEPLOY_PAYMASTER_SCRIPT_PARAMS, + }, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, +}; +use xshell::Shell; -use crate::messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}; -use crate::utils::forge::{check_the_balance, fill_forge_private_key}; +use crate::{ + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_PAYMASTER}, + utils::forge::{check_the_balance, fill_forge_private_key}, +}; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 2d1e6fddfc67..432b0045b3d3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -1,22 +1,20 @@ use std::path::PathBuf; use anyhow::Context; -use xshell::Shell; - use common::{ config::global_config, - db::{DatabaseConfig, drop_db_if_exists, init_db, migrate_db}, + db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, logger, spinner::Spinner, }; -use config::{ChainConfig, EcosystemConfig, SecretsConfig}; -use config::traits::{FileConfigWithDefaultName, ReadConfigWithBasePath, SaveConfigWithBasePath}; +use config::{traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig}; use types::ProverMode; +use xshell::Shell; +use super::args::genesis::GenesisArgsFinal; use crate::{ commands::chain::args::genesis::GenesisArgs, consts::{PROVER_MIGRATIONS, SERVER_MIGRATIONS}, - defaults::MAIN_ROCKS_DB_PREFIX, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, @@ -28,8 +26,6 @@ use crate::{ utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; -use super::args::genesis::GenesisArgsFinal; - pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -51,7 +47,8 @@ pub async fn genesis( ) -> anyhow::Result<()> { shell.create_dir(&config.rocks_db_path)?; - let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main).context("Failed to create rocks db path")?; + let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) + .context("Failed to create rocks db path")?; let mut general = config.get_general_config()?; general.set_rocks_db_config(rocks_db)?; if config.prover_version != ProverMode::NoProofs { @@ -60,9 +57,7 @@ pub async fn genesis( general.save_with_base_path(shell, &config.configs)?; let mut secrets = config.get_secrets_config()?; - secrets.set_databases( - &args.server_db, &args.prover_db, - ); + secrets.set_databases(&args.server_db, &args.prover_db); secrets.save_with_base_path(&shell, &config.configs)?; logger::note( @@ -83,7 +78,7 @@ pub async fn genesis( config.link_to_code.clone(), args.dont_drop, ) - .await?; + .await?; spinner.finish(); let spinner = Spinner::new(MSG_STARTING_GENESIS_SPINNER); @@ -116,7 +111,7 @@ async fn initialize_databases( path_to_server_migration, &server_db_config.full_url(), ) - .await?; + .await?; if global_config().verbose { logger::debug(MSG_INITIALIZING_PROVER_DATABASE) @@ -133,7 +128,7 @@ async fn initialize_databases( path_to_prover_migration, &prover_db_config.full_url(), ) - .await?; + .await?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index dd370b424883..a7f45b52e8ae 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -1,6 +1,4 @@ use anyhow::Context; -use xshell::{cmd, Shell}; - use common::{ cmd::Cmd, config::global_config, @@ -8,26 +6,30 @@ use common::{ logger, spinner::Spinner, }; -use config::{ChainConfig, ContractsConfig, copy_configs, EcosystemConfig, forge_interface::{ - register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, - script_params::REGISTER_CHAIN_SCRIPT_PARAMS, -}, traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}}; +use config::{ + copy_configs, + forge_interface::{ + register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, + script_params::REGISTER_CHAIN_SCRIPT_PARAMS, + }, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, +}; +use xshell::{cmd, Shell}; +use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ - args::init::InitArgs, deploy_paymaster, initialize_bridges, + args::init::InitArgs, deploy_paymaster, genesis::genesis, initialize_bridges, }, messages::{ - MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_INITIALIZED, - MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, - msg_initializing_chain, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, + msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, + MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_GENESIS_DATABASE_ERR, + MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; -use crate::commands::chain::genesis::genesis; -use crate::utils::forge::{check_the_balance, fill_forge_private_key}; - -use super::args::init::InitArgsFinal; pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); @@ -56,11 +58,11 @@ pub async fn init( build_l1_contracts(shell, ecosystem_config)?; let mut genesis_config = chain_config.get_genesis_config()?; - genesis_config.set_from_config(&chain_config); + genesis_config.set_from_chain_config(&chain_config); genesis_config.save_with_base_path(shell, &chain_config.configs)?; // Copy ecosystem contracts - let mut contracts_config = chain_config.get_contracts_config()?; + let mut contracts_config = ecosystem_config.get_contracts_config()?; contracts_config.l1.base_token_addr = chain_config.base_token.address; contracts_config.save_with_base_path(shell, &chain_config.configs)?; @@ -73,7 +75,7 @@ pub async fn init( &mut contracts_config, init_args.l1_rpc_url.clone(), ) - .await?; + .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); @@ -86,26 +88,33 @@ pub async fn init( &init_args.forge_args.clone(), init_args.l1_rpc_url.clone(), ) - .await?; + .await?; spinner.finish(); initialize_bridges::initialize_bridges( shell, chain_config, ecosystem_config, + &mut contracts_config, init_args.forge_args.clone(), ) - .await?; + .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; if init_args.deploy_paymaster { - deploy_paymaster::deploy_paymaster(shell, chain_config, &mut contracts_config, init_args.forge_args.clone()) - .await?; + deploy_paymaster::deploy_paymaster( + shell, + chain_config, + &mut contracts_config, + init_args.forge_args.clone(), + ) + .await?; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; } let mut secrets = chain_config.get_secrets_config()?; secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); secrets.save_with_base_path(shell, &chain_config.configs)?; - contracts_config.save_with_base_path(shell, &chain_config.configs)?; genesis(init_args.genesis_args.clone(), shell, chain_config) .await .context(MSG_GENESIS_DATABASE_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs index 1edba5387a6b..2fab4f8ae6d4 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/initialize_bridges.rs @@ -1,8 +1,6 @@ use std::path::Path; use anyhow::Context; -use xshell::{cmd, Shell}; - use common::{ cmd::Cmd, config::global_config, @@ -10,16 +8,19 @@ use common::{ spinner::Spinner, }; use config::{ - ChainConfig, - EcosystemConfig, forge_interface::{ initialize_bridges::{input::InitializeBridgeInput, output::InitializeBridgeOutput}, script_params::INITIALIZE_BRIDGES_SCRIPT_PARAMS, - }, traits::{ReadConfig, SaveConfig}, + }, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, }; +use xshell::{cmd, Shell}; -use crate::messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}; -use crate::utils::forge::{check_the_balance, fill_forge_private_key}; +use crate::{ + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_INITIALIZING_BRIDGES_SPINNER}, + utils::forge::{check_the_balance, fill_forge_private_key}, +}; pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { let chain_name = global_config().chain_name.clone(); @@ -28,8 +29,17 @@ pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { .load_chain(chain_name) .context(MSG_CHAIN_NOT_INITIALIZED)?; + let mut contracts = chain_config.get_contracts_config()?; let spinner = Spinner::new(MSG_INITIALIZING_BRIDGES_SPINNER); - initialize_bridges(shell, &chain_config, &ecosystem_config, args).await?; + initialize_bridges( + shell, + &chain_config, + &ecosystem_config, + &mut contracts, + args, + ) + .await?; + contracts.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); Ok(()) @@ -39,6 +49,7 @@ pub async fn initialize_bridges( shell: &Shell, chain_config: &ChainConfig, ecosystem_config: &EcosystemConfig, + contracts_config: &mut ContractsConfig, forge_args: ForgeScriptArgs, ) -> anyhow::Result<()> { build_l2_contracts(shell, &ecosystem_config.link_to_code)?; @@ -72,7 +83,7 @@ pub async fn initialize_bridges( INITIALIZE_BRIDGES_SCRIPT_PARAMS.output(&chain_config.link_to_code), )?; - update_l2_shared_bridge(shell, chain_config, &output)?; + contracts_config.set_l2_shared_bridge(&output)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs index 65e87010e84e..929b096b7813 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs @@ -1,22 +1,20 @@ use std::path::Path; use anyhow::Context; -use xshell::Shell; - use common::{config::global_config, logger}; use config::{ - ChainConfig, DatabaseSecrets, EcosystemConfig, external_node::ENConfig, - L1Secret, SecretsConfig, traits::SaveConfigWithBasePath, + external_node::ENConfig, traits::SaveConfigWithBasePath, ChainConfig, DatabaseSecrets, + EcosystemConfig, L1Secret, SecretsConfig, }; +use xshell::Shell; use crate::{ - commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal} - , + commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal}, messages::{ - MSG_CHAIN_NOT_INITIALIZED, msg_preparing_en_config_is_done, MSG_PREPARING_EN_CONFIGS, + msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; -use crate::utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}; pub fn run(shell: &Shell, args: PrepareConfigArgs) -> anyhow::Result<()> { logger::info(MSG_PREPARING_EN_CONFIGS); diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs index 49e8687eb809..3099b3cf8c27 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs @@ -4,20 +4,16 @@ use std::{ }; use anyhow::Context; -use xshell::{cmd, Shell}; - use common::{ cmd::Cmd, config::global_config, forge::{Forge, ForgeScriptArgs}, logger, - Prompt, spinner::Spinner, + Prompt, }; use config::{ - ChainConfig, - ContractsConfig, - EcosystemConfig, forge_interface::{ + forge_interface::{ deploy_ecosystem::{ input::{ DeployErc20Config, DeployL1Config, Erc20DeploymentConfig, InitialDeploymentConfig, @@ -25,13 +21,17 @@ use config::{ output::{DeployErc20Output, DeployL1Output}, }, script_params::{DEPLOY_ECOSYSTEM_SCRIPT_PARAMS, DEPLOY_ERC20_SCRIPT_PARAMS}, - }, GenesisConfig, traits::{ + }, + traits::{ FileConfigWithDefaultName, ReadConfig, ReadConfigWithBasePath, SaveConfig, SaveConfigWithBasePath, }, + ChainConfig, ContractsConfig, EcosystemConfig, GenesisConfig, }; use types::{L1Network, ProverMode, WalletCreation}; +use xshell::{cmd, Shell}; +use super::args::init::{EcosystemArgsFinal, EcosystemInitArgs, EcosystemInitArgsFinal}; use crate::{ accept_ownership::accept_owner, commands::{ @@ -42,16 +42,14 @@ use crate::{ }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, messages::{ - MSG_CHAIN_NOT_INITIALIZED, MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, + msg_ecosystem_initialized, msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED, + MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, MSG_DEPLOYING_ERC20_SPINNER, MSG_DISTRIBUTING_ETH_SPINNER, MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, - msg_ecosystem_initialized, msg_initializing_chain, MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER, }, + utils::forge::{check_the_balance, fill_forge_private_key}, }; -use crate::utils::forge::{check_the_balance, fill_forge_private_key}; - -use super::args::init::{EcosystemArgsFinal, EcosystemInitArgs, EcosystemInitArgsFinal}; pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; @@ -72,7 +70,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &ecosystem_config, &initial_deployment_config, ) - .await?; + .await?; if final_ecosystem_args.deploy_erc20 { logger::info(MSG_DEPLOYING_ERC20); @@ -88,7 +86,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { final_ecosystem_args.forge_args.clone(), final_ecosystem_args.ecosystem.l1_rpc_url.clone(), ) - .await?; + .await?; } // If the name of chain passed then we deploy exactly this chain otherwise deploy all chains @@ -116,7 +114,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &chain_config, final_ecosystem_args.ecosystem.l1_rpc_url.clone(), ) - .await?; + .await?; chain::init::init( &mut chain_init_args, @@ -124,7 +122,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { &ecosystem_config, &chain_config, ) - .await?; + .await?; } logger::outro(msg_ecosystem_initialized(&list_of_chains.join(","))); @@ -159,7 +157,7 @@ pub async fn distribute_eth( ecosystem_config.l1_network.chain_id(), AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, ) - .await?; + .await?; spinner.finish(); } Ok(()) @@ -183,7 +181,7 @@ async fn init( ecosystem_config, initial_deployment_config, ) - .await?; + .await?; contracts.save_with_base_path(shell, &ecosystem_config.config)?; Ok(contracts) } @@ -239,7 +237,7 @@ async fn deploy_ecosystem( initial_deployment_config, ecosystem.l1_rpc_url.clone(), ) - .await; + .await; } let ecosystem_contracts_path = match &ecosystem.ecosystem_contracts_path { @@ -334,7 +332,7 @@ async fn deploy_ecosystem_inner( &forge_args, l1_rpc_url.clone(), ) - .await?; + .await?; accept_owner( shell, @@ -345,7 +343,7 @@ async fn deploy_ecosystem_inner( &forge_args, l1_rpc_url.clone(), ) - .await?; + .await?; Ok(contracts_config) } diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index 32b7f603ee26..cbecb351d8d9 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -1,12 +1,11 @@ use clap::{command, Parser, Subcommand}; -use xshell::Shell; - use common::{ check_prerequisites, - config::{global_config, GlobalConfig, init_global_config}, + config::{global_config, init_global_config, GlobalConfig}, init_prompt_theme, logger, }; use config::EcosystemConfig; +use xshell::Shell; use crate::commands::{ args::{RunExternalNodeArgs, RunServerArgs}, diff --git a/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs index 17b751790cb7..fc80aca100bc 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/rocks_db.rs @@ -1,10 +1,11 @@ use std::path::Path; -use xshell::Shell; - use config::RocksDbs; +use xshell::Shell; -use crate::defaults::{EN_ROCKS_DB_PREFIX, MAIN_ROCKS_DB_PREFIX, ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE}; +use crate::defaults::{ + EN_ROCKS_DB_PREFIX, MAIN_ROCKS_DB_PREFIX, ROCKS_DB_STATE_KEEPER, ROCKS_DB_TREE, +}; pub enum RocksDBDirOption { Main, @@ -15,13 +16,19 @@ impl RocksDBDirOption { pub fn prefix(&self) -> &str { match self { RocksDBDirOption::Main => MAIN_ROCKS_DB_PREFIX, - RocksDBDirOption::ExternalNode => EN_ROCKS_DB_PREFIX + RocksDBDirOption::ExternalNode => EN_ROCKS_DB_PREFIX, } } } -pub fn recreate_rocksdb_dirs(shell: &Shell, rocks_db_path: &Path, option: RocksDBDirOption) -> anyhow::Result { - let state_keeper = rocks_db_path.join(option.prefix()).join(ROCKS_DB_STATE_KEEPER); +pub fn recreate_rocksdb_dirs( + shell: &Shell, + rocks_db_path: &Path, + option: RocksDBDirOption, +) -> anyhow::Result { + let state_keeper = rocks_db_path + .join(option.prefix()) + .join(ROCKS_DB_STATE_KEEPER); shell.remove_path(&state_keeper)?; let merkle_tree = rocks_db_path.join(option.prefix()).join(ROCKS_DB_TREE); shell.remove_path(&merkle_tree)?; From edc33e4164eb132517bbd52f945d3ffcde76137e Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 13:22:58 +0200 Subject: [PATCH 42/56] Update working with configs Signed-off-by: Danil --- zk_toolbox/crates/config/src/genesis.rs | 2 +- zk_toolbox/crates/zk_inception/src/commands/chain/init.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/config/src/genesis.rs b/zk_toolbox/crates/config/src/genesis.rs index a6dbb3dcc955..e666931870a8 100644 --- a/zk_toolbox/crates/config/src/genesis.rs +++ b/zk_toolbox/crates/config/src/genesis.rs @@ -22,7 +22,7 @@ pub struct GenesisConfig { } impl GenesisConfig { - pub fn set_from_chain_config(&mut self, config: &ChainConfig) { + pub fn update_from_chain_config(&mut self, config: &ChainConfig) { self.l2_chain_id = config.chain_id; self.l1_chain_id = config.l1_network.chain_id(); self.l1_batch_commit_data_generator_mode = Some(config.l1_batch_commit_data_generator_mode); diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index a7f45b52e8ae..7f4f1920bacd 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -58,7 +58,7 @@ pub async fn init( build_l1_contracts(shell, ecosystem_config)?; let mut genesis_config = chain_config.get_genesis_config()?; - genesis_config.set_from_chain_config(&chain_config); + genesis_config.update_from_chain_config(&chain_config); genesis_config.save_with_base_path(shell, &chain_config.configs)?; // Copy ecosystem contracts @@ -98,7 +98,7 @@ pub async fn init( &mut contracts_config, init_args.forge_args.clone(), ) - .await?; + .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; if init_args.deploy_paymaster { @@ -108,7 +108,7 @@ pub async fn init( &mut contracts_config, init_args.forge_args.clone(), ) - .await?; + .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; } From df823e462734a9b8efbf50d06c9ab6c187fbb92a Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 13:14:03 +0200 Subject: [PATCH 43/56] Refactor external node commands Signed-off-by: Danil --- zk_toolbox/crates/config/src/general.rs | 38 ++++++++++++ .../zk_inception/src/commands/args/mod.rs | 2 - .../src/commands/chain/args/mod.rs | 1 - .../zk_inception/src/commands/chain/init.rs | 9 ++- .../zk_inception/src/commands/chain/mod.rs | 8 +-- .../src/commands/external_node.rs | 61 ------------------- .../src/commands/external_node/args/mod.rs | 2 + .../args/prepare_configs.rs} | 0 .../args/run.rs} | 0 .../src/commands/external_node/init.rs | 47 ++++++++++++++ .../src/commands/external_node/mod.rs | 24 ++++++++ .../prepare_configs.rs} | 8 +-- .../src/commands/external_node/run.rs | 37 +++++++++++ zk_toolbox/crates/zk_inception/src/main.rs | 8 +-- .../crates/zk_inception/src/messages.rs | 2 + 15 files changed, 163 insertions(+), 84 deletions(-) delete mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs rename zk_toolbox/crates/zk_inception/src/commands/{chain/args/prepare_external_node_configs.rs => external_node/args/prepare_configs.rs} (100%) rename zk_toolbox/crates/zk_inception/src/commands/{args/run_external_node.rs => external_node/args/run.rs} (100%) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs rename zk_toolbox/crates/zk_inception/src/commands/{chain/prepare_en_config.rs => external_node/prepare_configs.rs} (84%) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 74575f64e8c6..eaabad21cb9f 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -24,6 +24,24 @@ impl GeneralConfig { self.db.merkle_tree.path = rocks_dbs.merkle_tree; Ok(()) } + + pub fn ports_config(&self) -> PortsConfig { + PortsConfig { + web3_json_rpc_http_port: self.api.web3_json_rpc.http_port, + web3_json_rpc_ws_port: self.api.web3_json_rpc.ws_port, + healthcheck_port: self.api.healthcheck.port, + merkle_tree_port: self.api.merkle_tree.port, + prometheus_listener_port: self.api.prometheus.listener_port, + } + } + + pub fn update_ports(&mut self, ports_config: &PortsConfig) { + self.api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; + self.api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port; + self.api.healthcheck.port = ports_config.healthcheck_port; + self.api.merkle_tree.port = ports_config.merkle_tree_port; + self.api.prometheus.listener_port = ports_config.prometheus_listener_port; + } } impl FileConfigWithDefaultName for GeneralConfig { @@ -124,3 +142,23 @@ pub struct MerkleTreeApiConfig { #[serde(flatten)] pub other: serde_json::Value, } + +pub struct PortsConfig { + pub web3_json_rpc_http_port: u16, + pub web3_json_rpc_ws_port: u16, + pub healthcheck_port: u16, + pub merkle_tree_port: u16, + pub prometheus_listener_port: u16, +} + +impl PortsConfig { + pub fn next_empty_ports_config(&self) -> PortsConfig { + Self { + web3_json_rpc_http_port: self.web3_json_rpc_http_port + 100, + web3_json_rpc_ws_port: self.web3_json_rpc_ws_port + 100, + healthcheck_port: self.healthcheck_port + 100, + merkle_tree_port: self.merkle_tree_port + 100, + prometheus_listener_port: self.prometheus_listener_port + 100, + } + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs index 0bb080cc6dc8..7b21015691b9 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs @@ -1,5 +1,3 @@ -pub use run_external_node::*; pub use run_server::*; -mod run_external_node; mod run_server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs index 4c5c1e1d4f74..08f39a90a843 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/mod.rs @@ -1,4 +1,3 @@ pub mod create; pub mod genesis; pub mod init; -pub mod prepare_external_node_configs; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 7f4f1920bacd..15751a4263ee 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -21,7 +21,10 @@ use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ - args::init::InitArgs, deploy_paymaster, genesis::genesis, initialize_bridges, + args::init::{InitArgs, InitArgsFinal}, + deploy_paymaster, + genesis::genesis, + initialize_bridges, }, messages::{ msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_BUILDING_L1_CONTRACTS, @@ -98,7 +101,7 @@ pub async fn init( &mut contracts_config, init_args.forge_args.clone(), ) - .await?; + .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; if init_args.deploy_paymaster { @@ -108,7 +111,7 @@ pub async fn init( &mut contracts_config, init_args.forge_args.clone(), ) - .await?; + .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index f2478111e3d3..aabb0d714c53 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -4,10 +4,7 @@ use common::forge::ForgeScriptArgs; pub(crate) use create::create_chain_inner; use xshell::Shell; -use crate::commands::chain::args::{ - create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs, - prepare_external_node_configs::PrepareConfigArgs, -}; +use crate::commands::chain::args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}; pub(crate) mod args; mod create; @@ -15,7 +12,6 @@ pub mod deploy_paymaster; pub mod genesis; pub(crate) mod init; mod initialize_bridges; -mod prepare_en_config; #[derive(Subcommand, Debug)] pub enum ChainCommands { @@ -29,7 +25,6 @@ pub enum ChainCommands { InitializeBridges(ForgeScriptArgs), /// Initialize bridges on l2 DeployPaymaster(ForgeScriptArgs), - PrepareExternalNodeConfig(PrepareConfigArgs), } pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<()> { @@ -39,6 +34,5 @@ pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<() ChainCommands::Genesis(args) => genesis::run(args, shell).await, ChainCommands::InitializeBridges(args) => initialize_bridges::run(args, shell).await, ChainCommands::DeployPaymaster(args) => deploy_paymaster::run(args, shell).await, - ChainCommands::PrepareExternalNodeConfig(args) => prepare_en_config::run(shell, args), } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node.rs deleted file mode 100644 index 7c9aa8b191aa..000000000000 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node.rs +++ /dev/null @@ -1,61 +0,0 @@ -use anyhow::Context; -use common::{ - config::global_config, - db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, - logger, - spinner::Spinner, -}; -use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; -use xshell::Shell; - -use crate::{ - commands::args::RunExternalNodeArgs, - consts::SERVER_MIGRATIONS, - external_node::RunExternalNode, - messages::{ - MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, - MSG_INITIALIZING_DATABASES_SPINNER, MSG_STARTING_SERVER, - }, -}; - -pub async fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { - let ecosystem_config = EcosystemConfig::from_file(shell)?; - - let chain = global_config().chain_name.clone(); - let chain_config = ecosystem_config - .load_chain(chain) - .context(MSG_CHAIN_NOT_INITIALIZED)?; - - logger::info(MSG_STARTING_SERVER); - - run_external_node(args, &chain_config, shell).await?; - - Ok(()) -} - -async fn run_external_node( - args: RunExternalNodeArgs, - chain_config: &ChainConfig, - shell: &Shell, -) -> anyhow::Result<()> { - if args.reinit { - let spin = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); - let secrets = SecretsConfig::read_with_base_path( - shell, - chain_config - .external_node_config_path - .clone() - .context("External node is not initalized")?, - )?; - let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; - drop_db_if_exists(&db_config) - .await - .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; - init_db(&db_config).await?; - let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); - migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; - spin.finish() - } - let server = RunExternalNode::new(args.components.clone(), chain_config)?; - server.run(shell, args.additional_args.clone()) -} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs new file mode 100644 index 000000000000..ebc7855c2b58 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/mod.rs @@ -0,0 +1,2 @@ +pub mod prepare_configs; +pub mod run; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs similarity index 100% rename from zk_toolbox/crates/zk_inception/src/commands/chain/args/prepare_external_node_configs.rs rename to zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs similarity index 100% rename from zk_toolbox/crates/zk_inception/src/commands/args/run_external_node.rs rename to zk_toolbox/crates/zk_inception/src/commands/external_node/args/run.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs new file mode 100644 index 000000000000..09a2e634f2e2 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs @@ -0,0 +1,47 @@ +use anyhow::Context; +use common::{ + config::global_config, + db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, + spinner::Spinner, +}; +use config::{traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, SecretsConfig}; +use xshell::Shell; + +use crate::{ + consts::SERVER_MIGRATIONS, + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, + MSG_INITIALIZING_DATABASES_SPINNER, + }, +}; + +pub async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + init(shell, &chain_config).await +} + +pub async fn init(shell: &Shell, chain_config: &ChainConfig) -> anyhow::Result<()> { + let spin = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); + let secrets = SecretsConfig::read_with_base_path( + shell, + chain_config + .external_node_config_path + .clone() + .context("External node is not initalized")?, + )?; + let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; + drop_db_if_exists(&db_config) + .await + .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; + init_db(&db_config).await?; + let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); + migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; + spin.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs new file mode 100644 index 000000000000..06e422de08b8 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/mod.rs @@ -0,0 +1,24 @@ +use args::{prepare_configs::PrepareConfigArgs, run::RunExternalNodeArgs}; +use clap::Parser; +use serde::{Deserialize, Serialize}; +use xshell::Shell; + +mod args; +mod init; +mod prepare_configs; +mod run; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub enum ExternalNodeCommands { + Configs(PrepareConfigArgs), + Init, + Run(RunExternalNodeArgs), +} + +pub async fn run(shell: &Shell, commands: ExternalNodeCommands) -> anyhow::Result<()> { + match commands { + ExternalNodeCommands::Configs(args) => prepare_configs::run(shell, args), + ExternalNodeCommands::Init => init::run(shell).await, + ExternalNodeCommands::Run(args) => run::run(shell, args).await, + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs similarity index 84% rename from zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs rename to zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 929b096b7813..64da678ff1fb 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/prepare_en_config.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -9,7 +9,7 @@ use config::{ use xshell::Shell; use crate::{ - commands::chain::args::prepare_external_node_configs::{PrepareConfigArgs, PrepareConfigFinal}, + commands::external_node::args::prepare_configs::{PrepareConfigArgs, PrepareConfigFinal}, messages::{ msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_PREPARING_EN_CONFIGS, }, @@ -55,11 +55,7 @@ fn prepare_configs( main_node_rate_limit_rps: None, }; let mut general_en = general.clone(); - general_en.api.web3_json_rpc.http_port = general_en.api.web3_json_rpc.http_port + 1000; - general_en.api.web3_json_rpc.ws_port = general_en.api.web3_json_rpc.ws_port + 1000; - general_en.api.healthcheck.port = general_en.api.healthcheck.port + 1000; - general_en.api.merkle_tree.port = general_en.api.merkle_tree.port + 1000; - general_en.api.prometheus.listener_port = general_en.api.prometheus.listener_port + 1000; + general_en.update_ports(&general.ports_config().next_empty_ports_config()); let secrets = SecretsConfig { database: DatabaseSecrets { server_url: args.db.full_url(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs new file mode 100644 index 000000000000..9d3da4663859 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/run.rs @@ -0,0 +1,37 @@ +use anyhow::Context; +use common::{config::global_config, logger}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::Shell; + +use crate::{ + commands::external_node::{args::run::RunExternalNodeArgs, init}, + external_node::RunExternalNode, + messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_EN}, +}; + +pub async fn run(shell: &Shell, args: RunExternalNodeArgs) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain = global_config().chain_name.clone(); + let chain_config = ecosystem_config + .load_chain(chain) + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + logger::info(MSG_STARTING_EN); + + run_external_node(args, &chain_config, shell).await?; + + Ok(()) +} + +async fn run_external_node( + args: RunExternalNodeArgs, + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result<()> { + if args.reinit { + init::init(shell, chain_config).await? + } + let server = RunExternalNode::new(args.components.clone(), chain_config)?; + server.run(shell, args.additional_args.clone()) +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index cbecb351d8d9..b4317ee86e92 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -8,9 +8,8 @@ use config::EcosystemConfig; use xshell::Shell; use crate::commands::{ - args::{RunExternalNodeArgs, RunServerArgs}, - chain::ChainCommands, - ecosystem::EcosystemCommands, + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, + external_node::ExternalNodeCommands, }; pub mod accept_ownership; @@ -45,7 +44,8 @@ pub enum InceptionSubcommands { /// Run server Server(RunServerArgs), // Run External Node - ExternalNode(RunExternalNodeArgs), + #[command(subcommand)] + ExternalNode(ExternalNodeCommands), /// Run containers for local development Containers, } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 2114c1998f31..816e1f96be1d 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -207,3 +207,5 @@ pub(super) fn msg_preparing_en_config_is_done(path: &Path) -> String { /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; + +pub(super) const MSG_STARTING_EN: &str = "Starting external node"; From 7f43e195ecf2ac0fcc6e8022fa1582769d85eb20 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 13:41:49 +0200 Subject: [PATCH 44/56] Add reinstall command Signed-off-by: Danil --- bin/zkt | 7 +++++++ zk_toolbox/crates/zk_inception/src/commands/chain/init.rs | 1 - zk_toolbox/crates/zk_inception/src/commands/mod.rs | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 bin/zkt diff --git a/bin/zkt b/bin/zkt new file mode 100755 index 000000000000..337ad5d73953 --- /dev/null +++ b/bin/zkt @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +cd $(dirname $0) +cd ../zk_toolbox + +cargo install --path ./crates/zk_inception --force +cargo install --path ./crates/zk_supervisor --force diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 15751a4263ee..67547f55f607 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -17,7 +17,6 @@ use config::{ }; use xshell::{cmd, Shell}; -use super::args::init::InitArgsFinal; use crate::{ accept_ownership::accept_admin, commands::chain::{ diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index 10fccd316eee..db34e1d8647d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -2,6 +2,6 @@ pub mod args; pub mod chain; pub mod containers; pub mod ecosystem; -pub mod prover; pub mod external_node; +pub mod prover; pub mod server; From bd4a380635357825b0e586042c54138192f3c57d Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 14:50:13 +0200 Subject: [PATCH 45/56] Update http_urls Signed-off-by: Danil --- zk_toolbox/crates/config/src/general.rs | 21 ++++++++++++++++++- .../commands/external_node/prepare_configs.rs | 2 +- zk_toolbox/crates/zk_inception/src/main.rs | 2 +- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index eaabad21cb9f..e1f3655d2200 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -1,6 +1,7 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; +use url::Url; use crate::{consts::GENERAL_FILE, traits::FileConfigWithDefaultName}; @@ -35,13 +36,31 @@ impl GeneralConfig { } } - pub fn update_ports(&mut self, ports_config: &PortsConfig) { + pub fn update_ports(&mut self, ports_config: &PortsConfig) -> anyhow::Result<()> { self.api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; + update_port_in_url( + &mut self.api.web3_json_rpc.http_url, + ports_config.web3_json_rpc_http_port, + )?; self.api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port; + update_port_in_url( + &mut self.api.web3_json_rpc.ws_url, + ports_config.web3_json_rpc_ws_port, + )?; self.api.healthcheck.port = ports_config.healthcheck_port; self.api.merkle_tree.port = ports_config.merkle_tree_port; self.api.prometheus.listener_port = ports_config.prometheus_listener_port; + Ok(()) + } +} + +fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> { + let mut http_url_url = Url::parse(&http_url)?; + if let Err(()) = http_url_url.set_port(Some(port)) { + anyhow::bail!("Wrong url, setting port is impossible"); } + *http_url = http_url_url.as_str().to_string(); + Ok(()) } impl FileConfigWithDefaultName for GeneralConfig { diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 64da678ff1fb..4df420474ecb 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -55,7 +55,7 @@ fn prepare_configs( main_node_rate_limit_rps: None, }; let mut general_en = general.clone(); - general_en.update_ports(&general.ports_config().next_empty_ports_config()); + general_en.update_ports(&general.ports_config().next_empty_ports_config())?; let secrets = SecretsConfig { database: DatabaseSecrets { server_url: args.db.full_url(), diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index b4317ee86e92..f381ad7fb47c 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -9,7 +9,7 @@ use xshell::Shell; use crate::commands::{ args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, - external_node::ExternalNodeCommands, + external_node::ExternalNodeCommands, prover::ProverCommands, }; pub mod accept_ownership; From 0bf6fe2c4123992326f9af890bf334a9a4140cb0 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 11:53:04 +0200 Subject: [PATCH 46/56] Integrate integration tests Signed-off-by: Danil --- core/tests/ts-integration/src/env.ts | 25 +++++++--- zk_toolbox/Cargo.lock | 1 + zk_toolbox/crates/zk_supervisor/Cargo.toml | 1 + .../src/commands/integration_tests.rs | 46 ++++++++++++++----- zk_toolbox/crates/zk_supervisor/src/main.rs | 8 +++- .../crates/zk_supervisor/src/messages.rs | 17 ++++++- 6 files changed, 77 insertions(+), 21 deletions(-) diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index c440e6b08ea6..bd8c40bbab60 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -57,11 +57,18 @@ function getMainWalletPk(pathToHome: string, network: string): string { */ async function loadTestEnvironmentFromFile(chain: string): Promise { const pathToHome = path.join(__dirname, '../../../..'); + let nodeMode; + if (process.env.EXTERNAL_NODE == 'true') { + nodeMode = NodeMode.External; + } else { + nodeMode = NodeMode.Main; + } let ecosystem = loadEcosystem(pathToHome); + // Genesis file is common for both EN and Main node + let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml', NodeMode.Main); - let generalConfig = loadConfig(pathToHome, chain, 'general.yaml'); - let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml'); - let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml'); + let generalConfig = loadConfig(pathToHome, chain, 'general.yaml', nodeMode); + let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml', nodeMode); const network = ecosystem.l1_network; let mainWalletPK = getMainWalletPk(pathToHome, network); @@ -115,8 +122,6 @@ async function loadTestEnvironmentFromFile(chain: string): Promise anyhow::Result<()> { +pub fn run( + shell: &Shell, + integration_test_commands: IntegrationTestCommands, +) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); - logger::info(MSG_INTEGRATION_TESTS_RUN_INFO); + logger::info(msg_integration_tests_run( + integration_test_commands.external_node, + )); build_repository(shell, &ecosystem_config)?; build_test_contracts(shell, &ecosystem_config)?; - Cmd::new( - cmd!(shell, "yarn jest --forceExit --testTimeout 60000") - .env("CHAIN_NAME", ecosystem_config.default_chain), - ) - .with_force_run() - .run()?; + let mut command = cmd!(shell, "yarn jest --forceExit --testTimeout 60000") + .env("CHAIN_NAME", ecosystem_config.default_chain); + + if integration_test_commands.external_node { + command = command.env( + "EXTERNAL_NODE", + format!("{:?}", integration_test_commands.external_node), + ) + } + if global_config().verbose { + command = command.env( + "ZKSYNC_DEBUG_LOGS", + format!("{:?}", global_config().verbose), + ) + } + + Cmd::new(command).with_force_run().run()?; logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS); diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index ab5629465a88..96ab59bdad10 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -12,6 +12,8 @@ use messages::{ }; use xshell::Shell; +use crate::commands::integration_tests::IntegrationTestCommands; + mod commands; mod dals; mod messages; @@ -30,7 +32,7 @@ enum SupervisorSubcommands { #[command(subcommand, about = MSG_SUBCOMMAND_DATABASE_ABOUT)] Database(DatabaseCommands), #[command(about = MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT)] - IntegrationTests, + IntegrationTests(IntegrationTestCommands), } #[derive(Parser, Debug)] @@ -93,7 +95,9 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { match args.command { SupervisorSubcommands::Database(command) => commands::database::run(shell, command).await?, - SupervisorSubcommands::IntegrationTests => commands::integration_tests::run(shell)?, + SupervisorSubcommands::IntegrationTests(args) => { + commands::integration_tests::run(shell, args)? + } } Ok(()) } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 31bdb0eb9b1d..39b2920b8c8f 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -1,5 +1,6 @@ // Ecosystem related messages pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; + pub(super) fn msg_global_chain_does_not_exist(chain: &str, available_chains: &str) -> String { format!("Chain with name {chain} doesnt exist, please choose one of: {available_chains}") } @@ -10,12 +11,15 @@ pub(super) const MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT: &str = "Run integration // Database related messages pub(super) const MSG_NO_DATABASES_SELECTED: &str = "No databases selected"; + pub(super) fn msg_database_info(gerund_verb: &str) -> String { format!("{gerund_verb} databases") } + pub(super) fn msg_database_success(past_verb: &str) -> String { format!("Databases {past_verb} successfully") } + pub(super) fn msg_database_loading(gerund_verb: &str, dal: &str) -> String { format!("{gerund_verb} database for dal {dal}...") } @@ -57,13 +61,24 @@ pub(super) const MSG_DATABASE_NEW_MIGRATION_DB_PROMPT: &str = "What database do you want to create a new migration for?"; pub(super) const MSG_DATABASE_NEW_MIGRATION_NAME_PROMPT: &str = "How do you want to name the migration?"; + pub(super) fn msg_database_new_migration_loading(dal: &str) -> String { format!("Creating new database migration for dal {}...", dal) } + pub(super) const MSG_DATABASE_NEW_MIGRATION_SUCCESS: &str = "Migration created successfully"; // Integration tests related messages -pub(super) const MSG_INTEGRATION_TESTS_RUN_INFO: &str = "Running integration tests"; + +pub(super) fn msg_integration_tests_run(external_node: bool) -> String { + let base = "Running integration tests"; + if external_node { + format!("{} for external node", base) + } else { + format!("{} for main server", base) + } +} + pub(super) const MSG_INTEGRATION_TESTS_RUN_SUCCESS: &str = "Integration tests ran successfully"; pub(super) const MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES: &str = "Building repository dependencies..."; From 6a46139060824caaf87c4c1e0c19564a6f401a51 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 24 Jun 2024 11:20:07 +0200 Subject: [PATCH 47/56] Fix general.yaml Signed-off-by: Danil --- etc/env/file_based/general.yaml | 2 +- yarn.lock | 686 +------------------------------- 2 files changed, 14 insertions(+), 674 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 9a671470bc74..3e7be109afbf 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -63,7 +63,7 @@ api: estimate_gas_scale_factor: 1.2 estimate_gas_acceptable_overestimation: 1000 max_tx_size: 1000000 - api_namespaces: [ eth,net,web3,zks,pubsub ] + api_namespaces: [ eth,net,web3,zks,pubsub,debug ] max_response_body_size_overrides: - method: eth_getTransactionReceipt # no size specified, meaning no size limit - method: zks_getProof diff --git a/yarn.lock b/yarn.lock index b7e2b98c431e..352ee1e57dc5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -333,360 +333,6 @@ resolved "https://registry.yarnpkg.com/@colors/colors/-/colors-1.5.0.tgz#bb504579c1cae923e6576a4f5da43d25f97bdbd9" integrity sha512-ooWCrlZP11i8GImSjTHYHLkvFDP48nS4+204nGb1RiX/WXYHmJA2III9/e2DWVabCESdW7hBAEzHRqUn9OUVvQ== -"@cspell/cspell-bundled-dicts@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-bundled-dicts/-/cspell-bundled-dicts-8.6.1.tgz#127b11ac24885aa4b725ab4ea6c0a0a18927e513" - integrity sha512-s6Av1xIgctYLuUiazKZjQ2WRUXc9dU38BOZXwM/lb7y8grQMEuTjST1c+8MOkZkppx48/sO7GHIF3k9rEzD3fg== - dependencies: - "@cspell/dict-ada" "^4.0.2" - "@cspell/dict-aws" "^4.0.1" - "@cspell/dict-bash" "^4.1.3" - "@cspell/dict-companies" "^3.0.31" - "@cspell/dict-cpp" "^5.1.3" - "@cspell/dict-cryptocurrencies" "^5.0.0" - "@cspell/dict-csharp" "^4.0.2" - "@cspell/dict-css" "^4.0.12" - "@cspell/dict-dart" "^2.0.3" - "@cspell/dict-django" "^4.1.0" - "@cspell/dict-docker" "^1.1.7" - "@cspell/dict-dotnet" "^5.0.0" - "@cspell/dict-elixir" "^4.0.3" - "@cspell/dict-en-common-misspellings" "^2.0.0" - "@cspell/dict-en-gb" "1.1.33" - "@cspell/dict-en_us" "^4.3.17" - "@cspell/dict-filetypes" "^3.0.3" - "@cspell/dict-fonts" "^4.0.0" - "@cspell/dict-fsharp" "^1.0.1" - "@cspell/dict-fullstack" "^3.1.5" - "@cspell/dict-gaming-terms" "^1.0.5" - "@cspell/dict-git" "^3.0.0" - "@cspell/dict-golang" "^6.0.5" - "@cspell/dict-haskell" "^4.0.1" - "@cspell/dict-html" "^4.0.5" - "@cspell/dict-html-symbol-entities" "^4.0.0" - "@cspell/dict-java" "^5.0.6" - "@cspell/dict-julia" "^1.0.1" - "@cspell/dict-k8s" "^1.0.2" - "@cspell/dict-latex" "^4.0.0" - "@cspell/dict-lorem-ipsum" "^4.0.0" - "@cspell/dict-lua" "^4.0.3" - "@cspell/dict-makefile" "^1.0.0" - "@cspell/dict-node" "^4.0.3" - "@cspell/dict-npm" "^5.0.15" - "@cspell/dict-php" "^4.0.6" - "@cspell/dict-powershell" "^5.0.3" - "@cspell/dict-public-licenses" "^2.0.6" - "@cspell/dict-python" "^4.1.11" - "@cspell/dict-r" "^2.0.1" - "@cspell/dict-ruby" "^5.0.2" - "@cspell/dict-rust" "^4.0.2" - "@cspell/dict-scala" "^5.0.0" - "@cspell/dict-software-terms" "^3.3.18" - "@cspell/dict-sql" "^2.1.3" - "@cspell/dict-svelte" "^1.0.2" - "@cspell/dict-swift" "^2.0.1" - "@cspell/dict-terraform" "^1.0.0" - "@cspell/dict-typescript" "^3.1.2" - "@cspell/dict-vue" "^3.0.0" - -"@cspell/cspell-json-reporter@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-json-reporter/-/cspell-json-reporter-8.6.1.tgz#d92e86a196d9f560cde49bd37139f7a9d8cc5ec3" - integrity sha512-75cmJgU9iQgrDnLFIUyvgybySJJi29BPw71z+8ZO9WhNofufxoSjaWepZeYV2nK0nHXM+MbdQG5Mmj/Lv6J1FA== - dependencies: - "@cspell/cspell-types" "8.6.1" - -"@cspell/cspell-pipe@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-pipe/-/cspell-pipe-8.6.1.tgz#b4ae588a331b0751be1e7e11211bcc3b54358233" - integrity sha512-guIlGhhOLQwfqevBSgp26b+SX4I1hCH+puAksWAk93bybKkcGtGpcavAQSN9qvamox4zcHnvGutEPF+UcXuceQ== - -"@cspell/cspell-resolver@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-resolver/-/cspell-resolver-8.6.1.tgz#0da1b57340cadf414b7416a065d1d166b4c521cc" - integrity sha512-ZUbYcvEhfokHG9qfUlIylUqEobG84PiDozCkE8U4h/rTSmYkf/nAD+M6yg+jQ0F2aTFGNbvpKKGFlfXFXveX7A== - dependencies: - global-directory "^4.0.1" - -"@cspell/cspell-service-bus@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-service-bus/-/cspell-service-bus-8.6.1.tgz#ea0b1f257de6de750ef3a4075aa0fbbfbdf92bce" - integrity sha512-WpI3fSW8t00UMetfd6tS8f9+xE3+ElIUO/bQ1YKK95TMIRdEUcH+QDxcHM66pJXEm4WiaN3H/MfWk1fIhGlJ8g== - -"@cspell/cspell-types@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/cspell-types/-/cspell-types-8.6.1.tgz#a1cfaa0f1412662733f75015992a97072b6d65ef" - integrity sha512-MXa9v6sXbbwyiNno7v7vczNph6AsMNWnpMRCcW3h/siXNQYRuMssdxqT5sQJ8Kurh3M/Wo7DlKX4n74elKL3iQ== - -"@cspell/dict-ada@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-ada/-/dict-ada-4.0.2.tgz#8da2216660aeb831a0d9055399a364a01db5805a" - integrity sha512-0kENOWQeHjUlfyId/aCM/mKXtkEgV0Zu2RhUXCBr4hHo9F9vph+Uu8Ww2b0i5a4ZixoIkudGA+eJvyxrG1jUpA== - -"@cspell/dict-aws@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-aws/-/dict-aws-4.0.1.tgz#a0e758531ae81792b928a3f406618296291a658a" - integrity sha512-NXO+kTPQGqaaJKa4kO92NAXoqS+i99dQzf3/L1BxxWVSBS3/k1f3uhmqIh7Crb/n22W793lOm0D9x952BFga3Q== - -"@cspell/dict-bash@^4.1.3": - version "4.1.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-bash/-/dict-bash-4.1.3.tgz#25fba40825ac10083676ab2c777e471c3f71b36e" - integrity sha512-tOdI3QVJDbQSwPjUkOiQFhYcu2eedmX/PtEpVWg0aFps/r6AyjUQINtTgpqMYnYuq8O1QUIQqnpx21aovcgZCw== - -"@cspell/dict-companies@^3.0.31": - version "3.0.31" - resolved "https://registry.yarnpkg.com/@cspell/dict-companies/-/dict-companies-3.0.31.tgz#f0dacabc5308096c0f12db8a8b802ece604d6bf7" - integrity sha512-hKVpV/lcGKP4/DpEPS8P4osPvFH/YVLJaDn9cBIOH6/HSmL5LbFgJNKpMGaYRbhm2FEX56MKE3yn/MNeNYuesQ== - -"@cspell/dict-cpp@^5.1.3": - version "5.1.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-cpp/-/dict-cpp-5.1.3.tgz#c0c34ccdecc3ff954877a56dbbf07a7bf53b218e" - integrity sha512-sqnriXRAInZH9W75C+APBh6dtben9filPqVbIsiRMUXGg+s02ekz0z6LbS7kXeJ5mD2qXoMLBrv13qH2eIwutQ== - -"@cspell/dict-cryptocurrencies@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-cryptocurrencies/-/dict-cryptocurrencies-5.0.0.tgz#19fbc7bdbec76ce64daf7d53a6d0f3cfff7d0038" - integrity sha512-Z4ARIw5+bvmShL+4ZrhDzGhnc9znaAGHOEMaB/GURdS/jdoreEDY34wdN0NtdLHDO5KO7GduZnZyqGdRoiSmYA== - -"@cspell/dict-csharp@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-csharp/-/dict-csharp-4.0.2.tgz#e55659dbe594e744d86b1baf0f3397fe57b1e283" - integrity sha512-1JMofhLK+4p4KairF75D3A924m5ERMgd1GvzhwK2geuYgd2ZKuGW72gvXpIV7aGf52E3Uu1kDXxxGAiZ5uVG7g== - -"@cspell/dict-css@^4.0.12": - version "4.0.12" - resolved "https://registry.yarnpkg.com/@cspell/dict-css/-/dict-css-4.0.12.tgz#59abf3512ae729835c933c38f64a3d8a5f09ce3d" - integrity sha512-vGBgPM92MkHQF5/2jsWcnaahOZ+C6OE/fPvd5ScBP72oFY9tn5GLuomcyO0z8vWCr2e0nUSX1OGimPtcQAlvSw== - -"@cspell/dict-dart@^2.0.3": - version "2.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-dart/-/dict-dart-2.0.3.tgz#75e7ffe47d5889c2c831af35acdd92ebdbd4cf12" - integrity sha512-cLkwo1KT5CJY5N5RJVHks2genFkNCl/WLfj+0fFjqNR+tk3tBI1LY7ldr9piCtSFSm4x9pO1x6IV3kRUY1lLiw== - -"@cspell/dict-data-science@^1.0.11": - version "1.0.11" - resolved "https://registry.yarnpkg.com/@cspell/dict-data-science/-/dict-data-science-1.0.11.tgz#4eabba75c21d27253c1114b4fbbade0ead739ffc" - integrity sha512-TaHAZRVe0Zlcc3C23StZqqbzC0NrodRwoSAc8dis+5qLeLLnOCtagYQeROQvDlcDg3X/VVEO9Whh4W/z4PAmYQ== - -"@cspell/dict-django@^4.1.0": - version "4.1.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-django/-/dict-django-4.1.0.tgz#2d4b765daf3c83e733ef3e06887ea34403a4de7a" - integrity sha512-bKJ4gPyrf+1c78Z0Oc4trEB9MuhcB+Yg+uTTWsvhY6O2ncFYbB/LbEZfqhfmmuK/XJJixXfI1laF2zicyf+l0w== - -"@cspell/dict-docker@^1.1.7": - version "1.1.7" - resolved "https://registry.yarnpkg.com/@cspell/dict-docker/-/dict-docker-1.1.7.tgz#bcf933283fbdfef19c71a642e7e8c38baf9014f2" - integrity sha512-XlXHAr822euV36GGsl2J1CkBIVg3fZ6879ZOg5dxTIssuhUOCiV2BuzKZmt6aIFmcdPmR14+9i9Xq+3zuxeX0A== - -"@cspell/dict-dotnet@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-dotnet/-/dict-dotnet-5.0.0.tgz#13690aafe14b240ad17a30225ac1ec29a5a6a510" - integrity sha512-EOwGd533v47aP5QYV8GlSSKkmM9Eq8P3G/eBzSpH3Nl2+IneDOYOBLEUraHuiCtnOkNsz0xtZHArYhAB2bHWAw== - -"@cspell/dict-elixir@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-elixir/-/dict-elixir-4.0.3.tgz#57c25843e46cf3463f97da72d9ef8e37c818296f" - integrity sha512-g+uKLWvOp9IEZvrIvBPTr/oaO6619uH/wyqypqvwpmnmpjcfi8+/hqZH8YNKt15oviK8k4CkINIqNhyndG9d9Q== - -"@cspell/dict-en-common-misspellings@^2.0.0": - version "2.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-en-common-misspellings/-/dict-en-common-misspellings-2.0.0.tgz#708f424d75dc65237a6fcb8d253bc1e7ab641380" - integrity sha512-NOg8dlv37/YqLkCfBs5OXeJm/Wcfb/CzeOmOZJ2ZXRuxwsNuolb4TREUce0yAXRqMhawahY5TSDRJJBgKjBOdw== - -"@cspell/dict-en-gb@1.1.33": - version "1.1.33" - resolved "https://registry.yarnpkg.com/@cspell/dict-en-gb/-/dict-en-gb-1.1.33.tgz#7f1fd90fc364a5cb77111b5438fc9fcf9cc6da0e" - integrity sha512-tKSSUf9BJEV+GJQAYGw5e+ouhEe2ZXE620S7BLKe3ZmpnjlNG9JqlnaBhkIMxKnNFkLY2BP/EARzw31AZnOv4g== - -"@cspell/dict-en_us@^4.3.17": - version "4.3.17" - resolved "https://registry.yarnpkg.com/@cspell/dict-en_us/-/dict-en_us-4.3.17.tgz#a39546b9ec4cc4fb1e9607575b2682b1155dda07" - integrity sha512-CS0Tb2f2YwQZ4VZ6+WLAO5uOzb0iO/iYSRl34kX4enq6quXxLYzwdfGAwv85wSYHPdga8tGiZFP+p8GPsi2JEg== - -"@cspell/dict-filetypes@^3.0.3": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-filetypes/-/dict-filetypes-3.0.3.tgz#ab0723ca2f4d3d5674e9c9745efc9f144e49c905" - integrity sha512-J9UP+qwwBLfOQ8Qg9tAsKtSY/WWmjj21uj6zXTI9hRLD1eG1uUOLcfVovAmtmVqUWziPSKMr87F6SXI3xmJXgw== - -"@cspell/dict-fonts@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-fonts/-/dict-fonts-4.0.0.tgz#9bc8beb2a7b068b4fdb45cb994b36fd184316327" - integrity sha512-t9V4GeN/m517UZn63kZPUYP3OQg5f0OBLSd3Md5CU3eH1IFogSvTzHHnz4Wqqbv8NNRiBZ3HfdY/pqREZ6br3Q== - -"@cspell/dict-fsharp@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-fsharp/-/dict-fsharp-1.0.1.tgz#d62c699550a39174f182f23c8c1330a795ab5f53" - integrity sha512-23xyPcD+j+NnqOjRHgW3IU7Li912SX9wmeefcY0QxukbAxJ/vAN4rBpjSwwYZeQPAn3fxdfdNZs03fg+UM+4yQ== - -"@cspell/dict-fullstack@^3.1.5": - version "3.1.5" - resolved "https://registry.yarnpkg.com/@cspell/dict-fullstack/-/dict-fullstack-3.1.5.tgz#35d18678161f214575cc613dd95564e05422a19c" - integrity sha512-6ppvo1dkXUZ3fbYn/wwzERxCa76RtDDl5Afzv2lijLoijGGUw5yYdLBKJnx8PJBGNLh829X352ftE7BElG4leA== - -"@cspell/dict-gaming-terms@^1.0.5": - version "1.0.5" - resolved "https://registry.yarnpkg.com/@cspell/dict-gaming-terms/-/dict-gaming-terms-1.0.5.tgz#d6ca40eb34a4c99847fd58a7354cd2c651065156" - integrity sha512-C3riccZDD3d9caJQQs1+MPfrUrQ+0KHdlj9iUR1QD92FgTOF6UxoBpvHUUZ9YSezslcmpFQK4xQQ5FUGS7uWfw== - -"@cspell/dict-git@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-git/-/dict-git-3.0.0.tgz#c275af86041a2b59a7facce37525e2af05653b95" - integrity sha512-simGS/lIiXbEaqJu9E2VPoYW1OTC2xrwPPXNXFMa2uo/50av56qOuaxDrZ5eH1LidFXwoc8HROCHYeKoNrDLSw== - -"@cspell/dict-golang@^6.0.5": - version "6.0.5" - resolved "https://registry.yarnpkg.com/@cspell/dict-golang/-/dict-golang-6.0.5.tgz#4dd2e2fda419730a21fb77ade3b90241ad4a5bcc" - integrity sha512-w4mEqGz4/wV+BBljLxduFNkMrd3rstBNDXmoX5kD4UTzIb4Sy0QybWCtg2iVT+R0KWiRRA56QKOvBsgXiddksA== - -"@cspell/dict-haskell@^4.0.1": - version "4.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-haskell/-/dict-haskell-4.0.1.tgz#e9fca7c452411ff11926e23ffed2b50bb9b95e47" - integrity sha512-uRrl65mGrOmwT7NxspB4xKXFUenNC7IikmpRZW8Uzqbqcu7ZRCUfstuVH7T1rmjRgRkjcIjE4PC11luDou4wEQ== - -"@cspell/dict-html-symbol-entities@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-html-symbol-entities/-/dict-html-symbol-entities-4.0.0.tgz#4d86ac18a4a11fdb61dfb6f5929acd768a52564f" - integrity sha512-HGRu+48ErJjoweR5IbcixxETRewrBb0uxQBd6xFGcxbEYCX8CnQFTAmKI5xNaIt2PKaZiJH3ijodGSqbKdsxhw== - -"@cspell/dict-html@^4.0.5": - version "4.0.5" - resolved "https://registry.yarnpkg.com/@cspell/dict-html/-/dict-html-4.0.5.tgz#03a5182148d80e6c25f71339dbb2b7c5b9894ef8" - integrity sha512-p0brEnRybzSSWi8sGbuVEf7jSTDmXPx7XhQUb5bgG6b54uj+Z0Qf0V2n8b/LWwIPJNd1GygaO9l8k3HTCy1h4w== - -"@cspell/dict-java@^5.0.6": - version "5.0.6" - resolved "https://registry.yarnpkg.com/@cspell/dict-java/-/dict-java-5.0.6.tgz#2462d6fc15f79ec15eb88ecf875b6ad2a7bf7a6a" - integrity sha512-kdE4AHHHrixyZ5p6zyms1SLoYpaJarPxrz8Tveo6gddszBVVwIUZ+JkQE1bWNLK740GWzIXdkznpUfw1hP9nXw== - -"@cspell/dict-julia@^1.0.1": - version "1.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-julia/-/dict-julia-1.0.1.tgz#900001417f1c4ea689530adfcc034c848458a0aa" - integrity sha512-4JsCLCRhhLMLiaHpmR7zHFjj1qOauzDI5ZzCNQS31TUMfsOo26jAKDfo0jljFAKgw5M2fEG7sKr8IlPpQAYrmQ== - -"@cspell/dict-k8s@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-k8s/-/dict-k8s-1.0.2.tgz#b19e66f4ac8a4264c0f3981ac6e23e88a60f1c91" - integrity sha512-tLT7gZpNPnGa+IIFvK9SP1LrSpPpJ94a/DulzAPOb1Q2UBFwdpFd82UWhio0RNShduvKG/WiMZf/wGl98pn+VQ== - -"@cspell/dict-latex@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-latex/-/dict-latex-4.0.0.tgz#85054903db834ea867174795d162e2a8f0e9c51e" - integrity sha512-LPY4y6D5oI7D3d+5JMJHK/wxYTQa2lJMSNxps2JtuF8hbAnBQb3igoWEjEbIbRRH1XBM0X8dQqemnjQNCiAtxQ== - -"@cspell/dict-lorem-ipsum@^4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-lorem-ipsum/-/dict-lorem-ipsum-4.0.0.tgz#2793a5dbfde474a546b0caecc40c38fdf076306e" - integrity sha512-1l3yjfNvMzZPibW8A7mQU4kTozwVZVw0AvFEdy+NcqtbxH+TvbSkNMqROOFWrkD2PjnKG0+Ea0tHI2Pi6Gchnw== - -"@cspell/dict-lua@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-lua/-/dict-lua-4.0.3.tgz#2d23c8f7e74b4e62000678d80e7d1ebb10b003e0" - integrity sha512-lDHKjsrrbqPaea13+G9s0rtXjMO06gPXPYRjRYawbNmo4E/e3XFfVzeci3OQDQNDmf2cPOwt9Ef5lu2lDmwfJg== - -"@cspell/dict-makefile@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-makefile/-/dict-makefile-1.0.0.tgz#5afb2910873ebbc01ab8d9c38661c4c93d0e5a40" - integrity sha512-3W9tHPcSbJa6s0bcqWo6VisEDTSN5zOtDbnPabF7rbyjRpNo0uHXHRJQF8gAbFzoTzBBhgkTmrfSiuyQm7vBUQ== - -"@cspell/dict-node@^4.0.3": - version "4.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-node/-/dict-node-4.0.3.tgz#5ae0222d72871e82978049f8e11ea627ca42fca3" - integrity sha512-sFlUNI5kOogy49KtPg8SMQYirDGIAoKBO3+cDLIwD4MLdsWy1q0upc7pzGht3mrjuyMiPRUV14Bb0rkVLrxOhg== - -"@cspell/dict-npm@^5.0.15": - version "5.0.15" - resolved "https://registry.yarnpkg.com/@cspell/dict-npm/-/dict-npm-5.0.15.tgz#c1d1646011fd0eb8ee119b481818a92223c459d1" - integrity sha512-sX0X5YWNW54F4baW7b5JJB6705OCBIZtUqjOghlJNORS5No7QY1IX1zc5FxNNu4gsaCZITAmfMi4ityXEsEThA== - -"@cspell/dict-php@^4.0.6": - version "4.0.6" - resolved "https://registry.yarnpkg.com/@cspell/dict-php/-/dict-php-4.0.6.tgz#fcdee4d850f279b2757eb55c4f69a3a221ac1f7e" - integrity sha512-ySAXisf7twoVFZqBV2o/DKiCLIDTHNqfnj0EfH9OoOUR7HL3rb6zJkm0viLUFDO2G/8SyIi6YrN/6KX+Scjjjg== - -"@cspell/dict-powershell@^5.0.3": - version "5.0.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-powershell/-/dict-powershell-5.0.3.tgz#7bceb4e7db39f87479a6d2af3a033ce26796ae49" - integrity sha512-lEdzrcyau6mgzu1ie98GjOEegwVHvoaWtzQnm1ie4DyZgMr+N6D0Iyj1lzvtmt0snvsDFa5F2bsYzf3IMKcpcA== - -"@cspell/dict-public-licenses@^2.0.6": - version "2.0.6" - resolved "https://registry.yarnpkg.com/@cspell/dict-public-licenses/-/dict-public-licenses-2.0.6.tgz#e6ac8e5cb3b0ef8503d67da14435ae86a875b6cc" - integrity sha512-bHqpSpJvLCUcWxj1ov/Ki8WjmESpYwRpQlqfdchekOTc93Huhvjm/RXVN1R4fVf4Hspyem1QVkCGqAmjJMj6sw== - -"@cspell/dict-python@^4.1.11": - version "4.1.11" - resolved "https://registry.yarnpkg.com/@cspell/dict-python/-/dict-python-4.1.11.tgz#4e339def01bf468b32d459c46ecb6894970b7eb8" - integrity sha512-XG+v3PumfzUW38huSbfT15Vqt3ihNb462ulfXifpQllPok5OWynhszCLCRQjQReV+dgz784ST4ggRxW452/kVg== - dependencies: - "@cspell/dict-data-science" "^1.0.11" - -"@cspell/dict-r@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-r/-/dict-r-2.0.1.tgz#73474fb7cce45deb9094ebf61083fbf5913f440a" - integrity sha512-KCmKaeYMLm2Ip79mlYPc8p+B2uzwBp4KMkzeLd5E6jUlCL93Y5Nvq68wV5fRLDRTf7N1LvofkVFWfDcednFOgA== - -"@cspell/dict-ruby@^5.0.2": - version "5.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-ruby/-/dict-ruby-5.0.2.tgz#cf1a71380c633dec0857143d3270cb503b10679a" - integrity sha512-cIh8KTjpldzFzKGgrqUX4bFyav5lC52hXDKo4LbRuMVncs3zg4hcSf4HtURY+f2AfEZzN6ZKzXafQpThq3dl2g== - -"@cspell/dict-rust@^4.0.2": - version "4.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-rust/-/dict-rust-4.0.2.tgz#e9111f0105ee6d836a1be8314f47347fd9f8fc3a" - integrity sha512-RhziKDrklzOntxAbY3AvNR58wnFGIo3YS8+dNeLY36GFuWOvXDHFStYw5Pod4f/VXbO/+1tXtywCC4zWfB2p1w== - -"@cspell/dict-scala@^5.0.0": - version "5.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-scala/-/dict-scala-5.0.0.tgz#b64365ad559110a36d44ccd90edf7151ea648022" - integrity sha512-ph0twaRoV+ylui022clEO1dZ35QbeEQaKTaV2sPOsdwIokABPIiK09oWwGK9qg7jRGQwVaRPEq0Vp+IG1GpqSQ== - -"@cspell/dict-software-terms@^3.3.18": - version "3.3.18" - resolved "https://registry.yarnpkg.com/@cspell/dict-software-terms/-/dict-software-terms-3.3.18.tgz#f25863c316eea195d74b170d41711e2c7402e9ca" - integrity sha512-LJZGGMGqS8KzgXJrSMs3T+6GoqHG9z8Bc+rqLzLzbtoR3FbsMasE9U8oP2PmS3q7jJLFjQkzmg508DrcuZuo2g== - -"@cspell/dict-sql@^2.1.3": - version "2.1.3" - resolved "https://registry.yarnpkg.com/@cspell/dict-sql/-/dict-sql-2.1.3.tgz#8d9666a82e35b310d0be4064032c0d891fbd2702" - integrity sha512-SEyTNKJrjqD6PAzZ9WpdSu6P7wgdNtGV2RV8Kpuw1x6bV+YsSptuClYG+JSdRExBTE6LwIe1bTklejUp3ZP8TQ== - -"@cspell/dict-svelte@^1.0.2": - version "1.0.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-svelte/-/dict-svelte-1.0.2.tgz#0c866b08a7a6b33bbc1a3bdbe6a1b484ca15cdaa" - integrity sha512-rPJmnn/GsDs0btNvrRBciOhngKV98yZ9SHmg8qI6HLS8hZKvcXc0LMsf9LLuMK1TmS2+WQFAan6qeqg6bBxL2Q== - -"@cspell/dict-swift@^2.0.1": - version "2.0.1" - resolved "https://registry.yarnpkg.com/@cspell/dict-swift/-/dict-swift-2.0.1.tgz#06ec86e52e9630c441d3c19605657457e33d7bb6" - integrity sha512-gxrCMUOndOk7xZFmXNtkCEeroZRnS2VbeaIPiymGRHj5H+qfTAzAKxtv7jJbVA3YYvEzWcVE2oKDP4wcbhIERw== - -"@cspell/dict-terraform@^1.0.0": - version "1.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-terraform/-/dict-terraform-1.0.0.tgz#c7b073bb3a03683f64cc70ccaa55ce9742c46086" - integrity sha512-Ak+vy4HP/bOgzf06BAMC30+ZvL9mzv21xLM2XtfnBLTDJGdxlk/nK0U6QT8VfFLqJ0ZZSpyOxGsUebWDCTr/zQ== - -"@cspell/dict-typescript@^3.1.2": - version "3.1.2" - resolved "https://registry.yarnpkg.com/@cspell/dict-typescript/-/dict-typescript-3.1.2.tgz#14d05f54db2984feaa24ea133b583d19c04cc104" - integrity sha512-lcNOYWjLUvDZdLa0UMNd/LwfVdxhE9rKA+agZBGjL3lTA3uNvH7IUqSJM/IXhJoBpLLMVEOk8v1N9xi+vDuCdA== - -"@cspell/dict-vue@^3.0.0": - version "3.0.0" - resolved "https://registry.yarnpkg.com/@cspell/dict-vue/-/dict-vue-3.0.0.tgz#68ccb432ad93fcb0fd665352d075ae9a64ea9250" - integrity sha512-niiEMPWPV9IeRBRzZ0TBZmNnkK3olkOPYxC1Ny2AX4TGlYRajcW0WUtoSHmvvjZNfWLSg2L6ruiBeuPSbjnG6A== - -"@cspell/dynamic-import@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/dynamic-import/-/dynamic-import-8.6.1.tgz#bc627779db48b39feb1536741534901c57e0a277" - integrity sha512-Fjvkcb5umIAcHfw/iiciYWgO2mXVuRZzQAWPSub6UFCxxcJlRz39YPXa+3O/m3lnXCeo8ChoaEN8qnuV4ogk6g== - dependencies: - import-meta-resolve "^4.0.0" - -"@cspell/strong-weak-map@8.6.1": - version "8.6.1" - resolved "https://registry.yarnpkg.com/@cspell/strong-weak-map/-/strong-weak-map-8.6.1.tgz#33c58f0d799624981399751dfb0c67328f0efdec" - integrity sha512-X6/7cy+GGVJFXsfrZapxVKn5mtehNTr7hTlg0bVj3iFoNYEPW9zq9l6WIcI4psmaU8G4DSrNsBK7pp87W3u16A== - "@cspotcode/source-map-support@^0.8.0": version "0.8.1" resolved "https://registry.yarnpkg.com/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz#00629c35a688e05a88b1cda684fb9d5e73f000a1" @@ -3290,11 +2936,6 @@ ansi-regex@^5.0.1: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== -ansi-regex@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-6.0.1.tgz#3183e38fae9a65d7cb5e53945cd5897d0260a06a" - integrity sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA== - ansi-styles@^3.2.1: version "3.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-3.2.1.tgz#41fbb20243e50b12be0f04b8dedbf07520ce841d" @@ -3393,11 +3034,6 @@ array-includes@^3.1.7: get-intrinsic "^1.2.4" is-string "^1.0.7" -array-timsort@^1.0.3: - version "1.0.3" - resolved "https://registry.yarnpkg.com/array-timsort/-/array-timsort-1.0.3.tgz#3c9e4199e54fb2b9c3fe5976396a21614ef0d926" - integrity sha512-/+3GRL7dDAGEfM6TseQk/U+mi18TU2Ms9I3UlLdUMhz2hbvGNTKdj9xniwXfUqgYhHxRx0+8UnKkvlNwVU+cWQ== - array-union@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/array-union/-/array-union-2.1.0.tgz#b798420adbeb1de828d84acd8a2e23d3efe85e8d" @@ -3896,7 +3532,7 @@ call-bind@^1.0.2, call-bind@^1.0.5, call-bind@^1.0.6, call-bind@^1.0.7: get-intrinsic "^1.2.4" set-function-length "^1.2.1" -callsites@^3.0.0, callsites@^3.1.0: +callsites@^3.0.0: version "3.1.0" resolved "https://registry.yarnpkg.com/callsites/-/callsites-3.1.0.tgz#b3630abd8943432f54b3f0519238e33cd7df2f73" integrity sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ== @@ -3953,13 +3589,6 @@ chai@^4.3.10, chai@^4.3.4, chai@^4.3.6: pathval "^1.1.1" type-detect "^4.0.8" -chalk-template@^1.1.0: - version "1.1.0" - resolved "https://registry.yarnpkg.com/chalk-template/-/chalk-template-1.1.0.tgz#ffc55db6dd745e9394b85327c8ac8466edb7a7b1" - integrity sha512-T2VJbcDuZQ0Tb2EWwSotMPJjgpy1/tGee1BTpUNsGZ/qgNjV2t7Mvu+d4600U564nbLesN1x2dPL+xii174Ekg== - dependencies: - chalk "^5.2.0" - chalk@4.1.2, chalk@^4.0.0, chalk@^4.1.0, chalk@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" @@ -3977,11 +3606,6 @@ chalk@^2.4.1, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^5.2.0, chalk@^5.3.0: - version "5.3.0" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-5.3.0.tgz#67c20a7ebef70e7f3970a01f90fa210cb6860385" - integrity sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w== - char-regex@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/char-regex/-/char-regex-1.0.2.tgz#d744358226217f981ed58f479b1d6bcc29545dcf" @@ -4067,14 +3691,6 @@ clean-stack@^2.0.0: resolved "https://registry.yarnpkg.com/clean-stack/-/clean-stack-2.2.0.tgz#ee8472dbb129e727b31e8a10a427dee9dfe4008b" integrity sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A== -clear-module@^4.1.2: - version "4.1.2" - resolved "https://registry.yarnpkg.com/clear-module/-/clear-module-4.1.2.tgz#5a58a5c9f8dccf363545ad7284cad3c887352a80" - integrity sha512-LWAxzHqdHsAZlPlEyJ2Poz6AIs384mPeqLVCru2p0BrP9G/kVGuhNyZYClLO6cXlnuJjzC8xtsJIuMjKqLXoAw== - dependencies: - parent-module "^2.0.0" - resolve-from "^5.0.0" - cli-boxes@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/cli-boxes/-/cli-boxes-2.2.1.tgz#ddd5035d25094fce220e9cab40a45840a440318f" @@ -4231,11 +3847,6 @@ commander@^10.0.0: resolved "https://registry.yarnpkg.com/commander/-/commander-10.0.1.tgz#881ee46b4f77d1c1dccc5823433aa39b022cbe06" integrity sha512-y4Mg2tXshplEbSGzx7amzPwKKOCGuoSRP/CjEdwwk0FOGlUbq6lKuoyDZTNZkmxHdJtp54hdfY/JUrdL7Xfdug== -commander@^12.0.0: - version "12.0.0" - resolved "https://registry.yarnpkg.com/commander/-/commander-12.0.0.tgz#b929db6df8546080adfd004ab215ed48cf6f2592" - integrity sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA== - commander@^2.19.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -4268,17 +3879,6 @@ commander@~9.4.1: resolved "https://registry.yarnpkg.com/commander/-/commander-9.4.1.tgz#d1dd8f2ce6faf93147295c0df13c7c21141cfbdd" integrity sha512-5EEkTNyHNGFPD2H+c/dXXfQZYa/scCKasxWcXJaWnNJ99pnQN9Vnmqow+p+PlFPE63Q6mThaZws1T+HxfpgtPw== -comment-json@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/comment-json/-/comment-json-4.2.3.tgz#50b487ebbf43abe44431f575ebda07d30d015365" - integrity sha512-SsxdiOf064DWoZLH799Ata6u7iV658A11PlWtZATDlXPpKGJnbJZ5Z24ybixAi+LUUqJ/GKowAejtC5GFUG7Tw== - dependencies: - array-timsort "^1.0.3" - core-util-is "^1.0.3" - esprima "^4.0.1" - has-own-prop "^2.0.0" - repeat-string "^1.6.1" - concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -4294,17 +3894,6 @@ concat-stream@^1.6.0, concat-stream@^1.6.2, concat-stream@~1.6.2: readable-stream "^2.2.2" typedarray "^0.0.6" -configstore@^6.0.0: - version "6.0.0" - resolved "https://registry.yarnpkg.com/configstore/-/configstore-6.0.0.tgz#49eca2ebc80983f77e09394a1a56e0aca8235566" - integrity sha512-cD31W1v3GqUlQvbBCGcXmd2Nj9SvLDOP1oQ0YFuLETufzSPaKp11rYBsSOm7rCsW3OnIRAFM3OxRhceaXNYHkA== - dependencies: - dot-prop "^6.0.1" - graceful-fs "^4.2.6" - unique-string "^3.0.0" - write-file-atomic "^3.0.3" - xdg-basedir "^5.0.1" - convert-source-map@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" @@ -4330,7 +3919,7 @@ core-util-is@1.0.2: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha512-3lqz5YjWTYnW6dlDa5TLaTCcShfar1e40rmcJVwCBJC6mWlFuj0eCHIElmG1g5kyuJ/GD+8Wn4FFCcz4gJPfaQ== -core-util-is@^1.0.3, core-util-is@~1.0.0: +core-util-is@~1.0.0: version "1.0.3" resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== @@ -4429,123 +4018,6 @@ crypto-js@^4.2.0: resolved "https://registry.yarnpkg.com/crypto-js/-/crypto-js-4.2.0.tgz#4d931639ecdfd12ff80e8186dba6af2c2e856631" integrity sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q== -crypto-random-string@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-4.0.0.tgz#5a3cc53d7dd86183df5da0312816ceeeb5bb1fc2" - integrity sha512-x8dy3RnvYdlUcPOjkEHqozhiwzKNSq7GcPuXFbnyMOCHxX8V3OgIg/pYuabl2sbUPfIJaeAQB7PMOK8DFIdoRA== - dependencies: - type-fest "^1.0.1" - -cspell-config-lib@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-config-lib/-/cspell-config-lib-8.6.1.tgz#951052d985756e684c540f92f8c6c4df25869519" - integrity sha512-I6LatgXJb8mxKFzIywO81TlUD/qWnUDrhB6yTUPdP90bwZcXMmGoCsZxhd2Rvl9fz5fWne0T839I1coShfm86g== - dependencies: - "@cspell/cspell-types" "8.6.1" - comment-json "^4.2.3" - yaml "^2.4.1" - -cspell-dictionary@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-dictionary/-/cspell-dictionary-8.6.1.tgz#c39a86ddd2ec5d31783414ff963db65c838177bc" - integrity sha512-0SfKPi1QoWbGpZ/rWMR7Jn0+GaQT9PAMLWjVOu66PUNUXI5f4oCTHpnZE1Xts+5VX8shZC3TAMHEgtgKuQn4RQ== - dependencies: - "@cspell/cspell-pipe" "8.6.1" - "@cspell/cspell-types" "8.6.1" - cspell-trie-lib "8.6.1" - fast-equals "^5.0.1" - gensequence "^7.0.0" - -cspell-gitignore@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-gitignore/-/cspell-gitignore-8.6.1.tgz#abb25f15ef25377cf0f071dba958635bd9ded4e8" - integrity sha512-3gtt351sSDfN826aMXTqGHVLz2lz9ZHr8uemImUc24Q+676sXkJM9lXzqP8PUqwGhLyt5qSf+9pt0ieNwQy/cA== - dependencies: - cspell-glob "8.6.1" - find-up-simple "^1.0.0" - -cspell-glob@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-glob/-/cspell-glob-8.6.1.tgz#6d80f703e9df15d0f63d3b36dcd5bc07ca908325" - integrity sha512-QjtngIR0XsUQLmHHDO86hps/JR5sRxSBwCvcsNCEmSdpdofLFc8cuxi3o33JWge7UAPBCQOLGfpA7/Wx31srmw== - dependencies: - micromatch "^4.0.5" - -cspell-grammar@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-grammar/-/cspell-grammar-8.6.1.tgz#d623475a0752b662769fc2a4de4745c25f7c0cbd" - integrity sha512-MaG0e/F0b2FnIRULCZ61JxEiJgTP/6rsbUoR5nG9X+WmJYItYmxC1F/FPPrVeTu+jJr/8O4pdnslE20pimHaCw== - dependencies: - "@cspell/cspell-pipe" "8.6.1" - "@cspell/cspell-types" "8.6.1" - -cspell-io@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-io/-/cspell-io-8.6.1.tgz#3b0fc769a609df8b027d3f189272f59ec3c0f642" - integrity sha512-ofxBB8QtUPvh/bOwKLYsqU1hwQCet8E98jkn/5f4jtG+/x5Zd80I0Ez+tlbjiBmrrQfOKh+i8ipfzHD8JtoreQ== - dependencies: - "@cspell/cspell-service-bus" "8.6.1" - -cspell-lib@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-lib/-/cspell-lib-8.6.1.tgz#691b1fc80c128eea3c4a24b59d20b1de95a912e2" - integrity sha512-kGeDUypRtThFT81IdUK7yU8eUwO5MYWj8pGQ0N8WFsqbCahJrUdcocceVSpnCX48W3CXu12DkqYG9kv5Umn7Xw== - dependencies: - "@cspell/cspell-bundled-dicts" "8.6.1" - "@cspell/cspell-pipe" "8.6.1" - "@cspell/cspell-resolver" "8.6.1" - "@cspell/cspell-types" "8.6.1" - "@cspell/dynamic-import" "8.6.1" - "@cspell/strong-weak-map" "8.6.1" - clear-module "^4.1.2" - comment-json "^4.2.3" - configstore "^6.0.0" - cspell-config-lib "8.6.1" - cspell-dictionary "8.6.1" - cspell-glob "8.6.1" - cspell-grammar "8.6.1" - cspell-io "8.6.1" - cspell-trie-lib "8.6.1" - fast-equals "^5.0.1" - gensequence "^7.0.0" - import-fresh "^3.3.0" - resolve-from "^5.0.0" - vscode-languageserver-textdocument "^1.0.11" - vscode-uri "^3.0.8" - -cspell-trie-lib@8.6.1: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell-trie-lib/-/cspell-trie-lib-8.6.1.tgz#7ff0e5992602808aa50d292bccd2b2e9484f5c28" - integrity sha512-iuJuAyWoqTH/TpFAR/ISJGQQoW3oiw54GyvXIucPoCJt/jgQONDuzqPW+skiLvcgcTbXCN9dutZTb2gImIkmpw== - dependencies: - "@cspell/cspell-pipe" "8.6.1" - "@cspell/cspell-types" "8.6.1" - gensequence "^7.0.0" - -cspell@^8.3.2: - version "8.6.1" - resolved "https://registry.yarnpkg.com/cspell/-/cspell-8.6.1.tgz#b3fd935c2bcbec64b47377a4de5b569ab50daa47" - integrity sha512-/Qle15v4IQe7tViSWX0+RCZJ2HJ4HUCZV9Z4uOVasNUz+DWCrxysNR+pfCRYuLX/6lQdqCM9QCR9GZc7a2KIVA== - dependencies: - "@cspell/cspell-json-reporter" "8.6.1" - "@cspell/cspell-pipe" "8.6.1" - "@cspell/cspell-types" "8.6.1" - "@cspell/dynamic-import" "8.6.1" - chalk "^5.3.0" - chalk-template "^1.1.0" - commander "^12.0.0" - cspell-gitignore "8.6.1" - cspell-glob "8.6.1" - cspell-io "8.6.1" - cspell-lib "8.6.1" - fast-glob "^3.3.2" - fast-json-stable-stringify "^2.1.0" - file-entry-cache "^8.0.0" - get-stdin "^9.0.0" - semver "^7.6.0" - strip-ansi "^7.1.0" - vscode-uri "^3.0.8" - dashdash@^1.12.0: version "1.14.1" resolved "https://registry.yarnpkg.com/dashdash/-/dashdash-1.14.1.tgz#853cfa0f7cbe2fed5de20326b8dd581035f6e2f0" @@ -4801,13 +4273,6 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" -dot-prop@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/dot-prop/-/dot-prop-6.0.1.tgz#fc26b3cf142b9e59b74dbd39ed66ce620c681083" - integrity sha512-tE7ztYzXHIeyvc7N+hR3oi7FIbf/NIjVP9hmAt3yMXzrQ072/fpjGLx2GxNxGxUl5V73MEqYzioOMoVhGMJ5cA== - dependencies: - is-obj "^2.0.0" - dotenv@^16.0.3: version "16.4.5" resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" @@ -5295,7 +4760,7 @@ esprima@2.7.x, esprima@^2.7.1: resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581" integrity sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A== -esprima@^4.0.0, esprima@^4.0.1: +esprima@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -5716,11 +5181,6 @@ fast-diff@^1.1.2, fast-diff@^1.2.0: resolved "https://registry.yarnpkg.com/fast-diff/-/fast-diff-1.3.0.tgz#ece407fa550a64d638536cd727e129c61616e0f0" integrity sha512-VxPP4NqbUjj6MaAOafWeUn2cXWLcCtljklUtZf0Ind4XQ+QPtmA0b18zZy0jIQx+ExRVCR/ZQpBmik5lXshNsw== -fast-equals@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/fast-equals/-/fast-equals-5.0.1.tgz#a4eefe3c5d1c0d021aeed0bc10ba5e0c12ee405d" - integrity sha512-WF1Wi8PwwSY7/6Kx0vKXtw8RwuSGoM1bvDaJbu7MxDlR1vovZjIAKrnzyrThgAjm6JDTu0fVgWXDlMGspodfoQ== - fast-glob@^3.0.3, fast-glob@^3.2.12, fast-glob@^3.2.9, fast-glob@^3.3.1, fast-glob@^3.3.2: version "3.3.2" resolved "https://registry.yarnpkg.com/fast-glob/-/fast-glob-3.3.2.tgz#a904501e57cfdd2ffcded45e99a54fef55e46129" @@ -5770,13 +5230,6 @@ file-entry-cache@^6.0.1: dependencies: flat-cache "^3.0.4" -file-entry-cache@^8.0.0: - version "8.0.0" - resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-8.0.0.tgz#7787bddcf1131bffb92636c69457bbc0edd6d81f" - integrity sha512-XXTUwCvisa5oacNGRP9SfNtYBNAMi+RPwBFmblZEF7N7swHYQS6/Zfk7SRwx4D5j3CH211YNRco1DEMNVfZCnQ== - dependencies: - flat-cache "^4.0.0" - fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -5799,11 +5252,6 @@ find-replace@^3.0.0: dependencies: array-back "^3.0.1" -find-up-simple@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/find-up-simple/-/find-up-simple-1.0.0.tgz#21d035fde9fdbd56c8f4d2f63f32fd93a1cfc368" - integrity sha512-q7Us7kcjj2VMePAa02hDAF6d+MzsdsAWEwYyOpwUtlerRBkOEPBCRZrAV4XfcSN8fHAgaD0hP7miwoay6DCprw== - find-up@5.0.0, find-up@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc" @@ -5836,14 +5284,6 @@ flat-cache@^3.0.4: keyv "^4.5.3" rimraf "^3.0.2" -flat-cache@^4.0.0: - version "4.0.1" - resolved "https://registry.yarnpkg.com/flat-cache/-/flat-cache-4.0.1.tgz#0ece39fcb14ee012f4b0410bd33dd9c1f011127c" - integrity sha512-f7ccFPK3SXFHpx15UIGyRJ/FJQctuKZ0zVuN3frBo4HnK3cay9VEW0R6yPYFHC0AgqhukPzKjq22t5DmAyqGyw== - dependencies: - flatted "^3.2.9" - keyv "^4.5.4" - flat@^5.0.2: version "5.0.2" resolved "https://registry.yarnpkg.com/flat/-/flat-5.0.2.tgz#8ca6fe332069ffa9d324c327198c598259ceb241" @@ -6013,11 +5453,6 @@ ganache@7.4.3: bufferutil "4.0.5" utf-8-validate "5.0.7" -gensequence@^7.0.0: - version "7.0.0" - resolved "https://registry.yarnpkg.com/gensequence/-/gensequence-7.0.0.tgz#bb6aedec8ff665e3a6c42f92823121e3a6ea7718" - integrity sha512-47Frx13aZh01afHJTB3zTtKIlFI6vWY+MYCN9Qpew6i52rfKjnhCF/l1YlC8UmEMvvntZZ6z4PiCcmyuedR2aQ== - gensync@^1.0.0-beta.2: version "1.0.0-beta.2" resolved "https://registry.yarnpkg.com/gensync/-/gensync-1.0.0-beta.2.tgz#32a6ee76c3d7f52d46b2b1ae5d93fea8580a25e0" @@ -6059,16 +5494,16 @@ get-stdin@=8.0.0: resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-8.0.0.tgz#cbad6a73feb75f6eeb22ba9e01f89aa28aa97a53" integrity sha512-sY22aA6xchAzprjyqmSEQv4UbAAzRN0L2dQB0NlN5acTTK9Don6nhoc3eAbUnpZiCANAMfd/+40kVdKfFygohg== -get-stdin@^9.0.0, get-stdin@~9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-9.0.0.tgz#3983ff82e03d56f1b2ea0d3e60325f39d703a575" - integrity sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA== - get-stdin@~5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-5.0.1.tgz#122e161591e21ff4c52530305693f20e6393a398" integrity sha512-jZV7n6jGE3Gt7fgSTJoz91Ak5MuTLwMwkoYdjxuJ/AmjIsE1UC03y/IWkZCQGEvVNS9qoRNwy5BCqxImv0FVeA== +get-stdin@~9.0.0: + version "9.0.0" + resolved "https://registry.yarnpkg.com/get-stdin/-/get-stdin-9.0.0.tgz#3983ff82e03d56f1b2ea0d3e60325f39d703a575" + integrity sha512-dVKBjfWisLAicarI2Sf+JuBE/DghV4UzNAVe9yhEJuzeREd3JhOTE9cUaJTeSa77fsbQUK3pcOpJfM59+VKZaA== + get-stream@^6.0.0: version "6.0.1" resolved "https://registry.yarnpkg.com/get-stream/-/get-stream-6.0.1.tgz#a262d8eef67aced57c2852ad6167526a43cbf7b7" @@ -6188,13 +5623,6 @@ glob@~8.0.3: minimatch "^5.0.1" once "^1.3.0" -global-directory@^4.0.1: - version "4.0.1" - resolved "https://registry.yarnpkg.com/global-directory/-/global-directory-4.0.1.tgz#4d7ac7cfd2cb73f304c53b8810891748df5e361e" - integrity sha512-wHTUcDUoZ1H5/0iVqEudYW4/kAlN5cZ3j/bXn0Dpbizl9iaUVeWSHqiOjsgk6OW2bkLclbBjzewBz6weQ1zA2Q== - dependencies: - ini "4.1.1" - global-modules@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/global-modules/-/global-modules-2.0.0.tgz#997605ad2345f27f51539bea26574421215c7780" @@ -6263,7 +5691,7 @@ gopd@^1.0.1: dependencies: get-intrinsic "^1.1.3" -graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.6, graceful-fs@^4.2.9: +graceful-fs@^4.1.2, graceful-fs@^4.1.6, graceful-fs@^4.1.9, graceful-fs@^4.2.0, graceful-fs@^4.2.4, graceful-fs@^4.2.9: version "4.2.11" resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.11.tgz#4183e4e8bf08bb6e05bbb2f7d2e0c8f712ca40e3" integrity sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ== @@ -6400,11 +5828,6 @@ has-flag@^4.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== -has-own-prop@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/has-own-prop/-/has-own-prop-2.0.0.tgz#f0f95d58f65804f5d218db32563bb85b8e0417af" - integrity sha512-Pq0h+hvsVm6dDEa8x82GnLSYHOzNDt7f0ddFa3FqcQlgzEiptPqL+XrOJNavjOzSYiYWIrgeVYYgGlLmnxwilQ== - has-property-descriptors@^1.0.0, has-property-descriptors@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz#963ed7d071dc7bf5f084c5bfbe0d1b6222586854" @@ -6595,11 +6018,6 @@ import-local@^3.0.2: pkg-dir "^4.2.0" resolve-cwd "^3.0.0" -import-meta-resolve@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz#0b1195915689f60ab00f830af0f15cc841e8919e" - integrity sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA== - imurmurhash@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/imurmurhash/-/imurmurhash-0.1.4.tgz#9218b9b2b928a238b13dc4fb6b6d576f231453ea" @@ -6628,11 +6046,6 @@ inherits@2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" integrity sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw== -ini@4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/ini/-/ini-4.1.1.tgz#d95b3d843b1e906e56d6747d5447904ff50ce7a1" - integrity sha512-QQnnxNyfvmHFIsj7gkPcYymR8Jdw/o7mp5ZFihxn6h8Ci6fh3Dx4E1gPjpQEpIuPo9XVNY/ZUwh4BPMjGyL01g== - ini@^1.3.5, ini@~1.3.0: version "1.3.8" resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.8.tgz#a29da425b48806f34767a4efce397269af28432c" @@ -6803,11 +6216,6 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== -is-obj@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/is-obj/-/is-obj-2.0.0.tgz#473fb05d973705e3fd9620545018ca8e22ef4982" - integrity sha512-drqDG3cbczxxEJRoOXcOjtdp1J/lyp1mNn0xaznRs8+muBhgQcrnbspox5X5fOw0HnMnbfDzvnEMEtqDEJEo8w== - is-path-inside@^3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/is-path-inside/-/is-path-inside-3.0.3.tgz#d231362e53a07ff2b0e0ea7fed049161ffd16283" @@ -6859,7 +6267,7 @@ is-typed-array@^1.1.13: dependencies: which-typed-array "^1.1.14" -is-typedarray@^1.0.0, is-typedarray@~1.0.0: +is-typedarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a" integrity sha512-cyA56iCMHAh5CdzjJIa4aohJyeO1YbwLi3Jc35MmRU6poroFjIGZzUzupGiRPOjgHg9TLu43xbpwXk523fMxKA== @@ -7541,7 +6949,7 @@ keccak@^3.0.0, keccak@^3.0.2: node-gyp-build "^4.2.0" readable-stream "^3.6.0" -keyv@^4.5.3, keyv@^4.5.4: +keyv@^4.5.3: version "4.5.4" resolved "https://registry.yarnpkg.com/keyv/-/keyv-4.5.4.tgz#a879a99e29452f942439f2a405e3af8b31d4de93" integrity sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw== @@ -8043,7 +7451,7 @@ micro-ftch@^0.3.1: resolved "https://registry.yarnpkg.com/micro-ftch/-/micro-ftch-0.3.1.tgz#6cb83388de4c1f279a034fb0cf96dfc050853c5f" integrity sha512-/0LLxhzP0tfiR5hcQebtudP56gUurs2CLkGarnCiB/OqEyUFQ6U3paQi/tgLv0hBJYt2rnr9MNpxz4fiiugstg== -micromatch@^4.0.4, micromatch@^4.0.5: +micromatch@^4.0.4: version "4.0.5" resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6" integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA== @@ -8608,13 +8016,6 @@ parent-module@^1.0.0: dependencies: callsites "^3.0.0" -parent-module@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/parent-module/-/parent-module-2.0.0.tgz#fa71f88ff1a50c27e15d8ff74e0e3a9523bf8708" - integrity sha512-uo0Z9JJeWzv8BG+tRcapBKNJ0dro9cLyczGzulS6EfeyAdeC9sbojtW6XwvYxJkEne9En+J2XEl4zyglVeIwFg== - dependencies: - callsites "^3.1.0" - parse-cache-control@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/parse-cache-control/-/parse-cache-control-1.0.1.tgz#8eeab3e54fa56920fe16ba38f77fa21aacc2d74e" @@ -9223,11 +8624,6 @@ regexpp@^3.1.0: resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== -repeat-string@^1.6.1: - version "1.6.1" - resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== - req-cwd@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/req-cwd/-/req-cwd-2.0.0.tgz#d4082b4d44598036640fb73ddea01ed53db49ebc" @@ -9512,7 +8908,7 @@ semver@^6.3.0, semver@^6.3.1: resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" integrity sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA== -semver@^7.2.1, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.5.1, semver@^7.5.2, semver@^7.5.3, semver@^7.5.4, semver@^7.6.0: +semver@^7.2.1, semver@^7.3.4, semver@^7.3.5, semver@^7.3.7, semver@^7.5.1, semver@^7.5.2, semver@^7.5.3, semver@^7.5.4: version "7.6.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== @@ -10007,13 +9403,6 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: dependencies: ansi-regex "^5.0.1" -strip-ansi@^7.1.0: - version "7.1.0" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" - integrity sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ== - dependencies: - ansi-regex "^6.0.1" - strip-bom@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" @@ -10475,11 +9864,6 @@ type-fest@^0.7.1: resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.7.1.tgz#8dda65feaf03ed78f0a3f9678f1869147f7c5c48" integrity sha512-Ne2YiiGN8bmrmJJEuTWTLJR32nh/JdL1+PSicowtNb0WFpn59GK8/lfD61bVtzguz7b3PBt74nxpv/Pw5po5Rg== -type-fest@^1.0.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-1.4.0.tgz#e9fb813fe3bf1744ec359d55d1affefa76f14be1" - integrity sha512-yGSza74xk0UG8k+pLh5oeoYirvIiWo5t0/o3zHHAO2tRDiZcxWP7fywNlXhqb6/r6sWvwi+RsyQMWhVLe4BVuA== - typechain@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/typechain/-/typechain-4.0.3.tgz#e8fcd6c984676858c64eeeb155ea783a10b73779" @@ -10553,13 +9937,6 @@ typed-array-length@^1.0.6: is-typed-array "^1.1.13" possible-typed-array-names "^1.0.0" -typedarray-to-buffer@^3.1.5: - version "3.1.5" - resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080" - integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q== - dependencies: - is-typedarray "^1.0.0" - typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" @@ -10622,13 +9999,6 @@ undici@^5.14.0: dependencies: "@fastify/busboy" "^2.0.0" -unique-string@^3.0.0: - version "3.0.0" - resolved "https://registry.yarnpkg.com/unique-string/-/unique-string-3.0.0.tgz#84a1c377aff5fd7a8bc6b55d8244b2bd90d75b9a" - integrity sha512-VGXBUVwxKMBUznyffQweQABPRRW1vHZAbadFZud4pLFAqRGvv/96vafgjWFqzourzr8YonlQiPgH0YCJfawoGQ== - dependencies: - crypto-random-string "^4.0.0" - universalify@^0.1.0: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" @@ -10742,16 +10112,6 @@ verror@1.10.0: core-util-is "1.0.2" extsprintf "^1.2.0" -vscode-languageserver-textdocument@^1.0.11: - version "1.0.11" - resolved "https://registry.yarnpkg.com/vscode-languageserver-textdocument/-/vscode-languageserver-textdocument-1.0.11.tgz#0822a000e7d4dc083312580d7575fe9e3ba2e2bf" - integrity sha512-X+8T3GoiwTVlJbicx/sIAF+yuJAqz8VvwJyoMVhwEMoEKE/fkDmrqUgDMyBECcM2A2frVZIUj5HI/ErRXCfOeA== - -vscode-uri@^3.0.8: - version "3.0.8" - resolved "https://registry.yarnpkg.com/vscode-uri/-/vscode-uri-3.0.8.tgz#1770938d3e72588659a172d0fd4642780083ff9f" - integrity sha512-AyFQ0EVmsOZOlAnxoFOGOq1SQDWAB7C6aqMGS23svWAllfOaxbuFvcT8D1i8z3Gyn8fraVeZNNmN6e9bxxXkKw== - walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" @@ -10876,16 +10236,6 @@ wrappy@1: resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== -write-file-atomic@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-3.0.3.tgz#56bd5c5a5c70481cd19c571bd39ab965a5de56e8" - integrity sha512-AvHcyZ5JnSfq3ioSyjrBkH9yW4m7Ayk8/9My/DD9onKeu/94fwrMocemO2QAJFAlnnDN+ZDS+ZjAR5ua1/PV/Q== - dependencies: - imurmurhash "^0.1.4" - is-typedarray "^1.0.0" - signal-exit "^3.0.2" - typedarray-to-buffer "^3.1.5" - write-file-atomic@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/write-file-atomic/-/write-file-atomic-4.0.2.tgz#a9df01ae5b77858a027fd2e80768ee433555fcfd" @@ -10904,11 +10254,6 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -xdg-basedir@^5.0.1: - version "5.1.0" - resolved "https://registry.yarnpkg.com/xdg-basedir/-/xdg-basedir-5.1.0.tgz#1efba19425e73be1bc6f2a6ceb52a3d2c884c0c9" - integrity sha512-GCPAHLvrIH13+c0SuacwvRYj2SxJXQ4kaVTT5xgL3kPrz56XxkF21IGhjSE1+W0aw7gpBWRGXLCPnPby6lSpmQ== - xhr2@0.1.3: version "0.1.3" resolved "https://registry.yarnpkg.com/xhr2/-/xhr2-0.1.3.tgz#cbfc4759a69b4a888e78cf4f20b051038757bd11" @@ -10939,11 +10284,6 @@ yallist@^4.0.0: resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== -yaml@^2.4.1: - version "2.4.1" - resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.1.tgz#2e57e0b5e995292c25c75d2658f0664765210eed" - integrity sha512-pIXzoImaqmfOrL7teGUBt/T7ZDnyeGBWyXQBvOVhLkWLN37GXv8NMLK406UY6dS51JfcQHsmcW5cJ441bHg6Lg== - yaml@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.4.2.tgz#7a2b30f2243a5fc299e1f14ca58d475ed4bc5362" From aacf3699034c27b4a6cad1ad951118107704c9ef Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 24 Jun 2024 12:04:23 +0200 Subject: [PATCH 48/56] Fix contracts deployment Signed-off-by: Danil --- etc/env/file_based/contracts.yaml | 4 ---- zk_toolbox/crates/zk_inception/src/commands/chain/init.rs | 7 ++++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/etc/env/file_based/contracts.yaml b/etc/env/file_based/contracts.yaml index 4777cc56ab8f..e6f175a2727a 100644 --- a/etc/env/file_based/contracts.yaml +++ b/etc/env/file_based/contracts.yaml @@ -1,5 +1,3 @@ -create2_factory_addr: 0x0000000000000000000000000000000000000000 -create2_factory_salt: 0x7a084e42e7c4851b6efbe654f7955cca0f1e3a09ed32d56c569ca16044b52a3c l1: diamond_proxy_addr: "0x0000000000000000000000000000000000000000" default_upgrade_addr: "0x0000000000000000000000000000000000000000" @@ -20,5 +18,3 @@ ecosystem_contracts: state_transition_proxy_addr: "0x0000000000000000000000000000000000000000" bridgehub_proxy_addr: "0x0000000000000000000000000000000000000000" transparent_proxy_admin_addr: "0x0000000000000000000000000000000000000000" - validator_timelock_addr: "0x0000000000000000000000000000000000000000" - diamond_cut_data: "0x" diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 67547f55f607..9660e30da15f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -68,6 +68,10 @@ pub async fn init( contracts_config.l1.base_token_addr = chain_config.base_token.address; contracts_config.save_with_base_path(shell, &chain_config.configs)?; + let mut secrets = chain_config.get_secrets_config()?; + secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); + secrets.save_with_base_path(shell, &chain_config.configs)?; + let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); register_chain( shell, @@ -114,9 +118,6 @@ pub async fn init( contracts_config.save_with_base_path(shell, &chain_config.configs)?; } - let mut secrets = chain_config.get_secrets_config()?; - secrets.set_l1_rpc_url(init_args.l1_rpc_url.clone()); - secrets.save_with_base_path(shell, &chain_config.configs)?; genesis(init_args.genesis_args.clone(), shell, chain_config) .await .context(MSG_GENESIS_DATABASE_ERR)?; From 82021774449b594e46fd372c95e9207f8d7e6a19 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 24 Jun 2024 12:44:13 +0200 Subject: [PATCH 49/56] remove verbose Signed-off-by: Danil --- .github/workflows/ci-zk-toolbox-reusable.yml | 8 ++++---- contracts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 66e54bfa98a4..9cda3bd0f393 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -11,7 +11,7 @@ jobs: uses: ./.github/workflows/ci-core-lint-reusable.yml build: - runs-on: [matterlabs-ci-runner] + runs-on: [ matterlabs-ci-runner ] steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 @@ -48,8 +48,8 @@ jobs: compression-level: 0 integration_test: - runs-on: [matterlabs-ci-runner] - needs: [build] + runs-on: [ matterlabs-ci-runner ] + needs: [ build ] steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 @@ -90,7 +90,7 @@ jobs: - name: Run server run: | - ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log & + ci_run zk_inception server --ignore-prerequisites &>server.log & ci_run sleep 5 - name: Run integration tests diff --git a/contracts b/contracts index 8a70bbbc4812..db9387690502 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 8a70bbbc48125f5bde6189b4e3c6a3ee79631678 +Subproject commit db9387690502937de081a959b164db5a5262ce0a From 573605869c4659c5ee98c6f0396adbd2b06cbce8 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 24 Jun 2024 15:20:01 +0200 Subject: [PATCH 50/56] Addd external node init Signed-off-by: Danil --- .github/workflows/ci-zk-toolbox-reusable.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 9cda3bd0f393..c04c83960355 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -97,6 +97,21 @@ jobs: run: | ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose + + - name: Run external node server + run: | + ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \ + --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 + + ci_run zk_inception external-node run --reinit --ignore-prerequisites + ci_run zk_inception server --ignore-prerequisites &>server.log & + ci_run sleep 5 + + - name: Run integration tests en + run: | + ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node + + - name: Show server.log logs if: always() run: ci_run cat server.log || true From 343d737ad0ea43d0b572dad8c99dd58f2ebdd95c Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 24 Jun 2024 15:47:13 +0200 Subject: [PATCH 51/56] Fix ci Signed-off-by: Danil --- .github/workflows/ci-zk-toolbox-reusable.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index c04c83960355..76e2e2e317cb 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -102,9 +102,8 @@ jobs: run: | ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \ --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 - - ci_run zk_inception external-node run --reinit --ignore-prerequisites - ci_run zk_inception server --ignore-prerequisites &>server.log & + ci_run zk_inception external-node init --ignore-prerequisites + ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log & ci_run sleep 5 - name: Run integration tests en @@ -115,3 +114,7 @@ jobs: - name: Show server.log logs if: always() run: ci_run cat server.log || true + - name: Show external_node.log logs + if: always() + run: ci_run cat external_node.log || true + From 4cf13d0e92c88c4528d08610cbb61bd69db79eca Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 25 Jun 2024 15:06:29 +0200 Subject: [PATCH 52/56] Fix slugify Signed-off-by: Danil --- .../external_node/args/prepare_configs.rs | 16 ++++++++++------ zk_toolbox/crates/zk_supervisor/src/dals.rs | 8 ++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs index bcfd5769a370..e82fbd7ca155 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/args/prepare_configs.rs @@ -1,7 +1,8 @@ use clap::Parser; -use common::{db::DatabaseConfig, slugify, Prompt}; +use common::{db::DatabaseConfig, Prompt}; use config::ChainConfig; use serde::{Deserialize, Serialize}; +use slugify_rs::slugify; use url::Url; use crate::{ @@ -39,11 +40,14 @@ impl PrepareConfigArgs { .default(DATABASE_SERVER_URL.as_str()) .ask() }); - let db_name = slugify(&self.db_name.unwrap_or_else(|| { - Prompt::new(&msg_external_node_db_name_prompt(&chain_name)) - .default(&db_name) - .ask() - })); + let db_name = slugify!( + &self.db_name.unwrap_or_else(|| { + Prompt::new(&msg_external_node_db_name_prompt(&chain_name)) + .default(&db_name) + .ask() + }), + separator = "_" + ); let l1_rpc_url = self.l1_rpc_url.unwrap_or_else(|| { Prompt::new(&MSG_L1_RPC_URL_PROMPT) .default(&LOCAL_RPC_URL) diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index 7f53770bc012..90a8da3ec23c 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -1,4 +1,4 @@ -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use common::config::global_config; use config::{EcosystemConfig, SecretsConfig}; use url::Url; @@ -46,7 +46,11 @@ pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { Ok(Dal { path: PROVER_DAL_PATH.to_string(), - url: secrets.database.prover_url.unwrap().clone(), + url: secrets + .database + .prover_url + .context("Prover url must be presented")? + .clone(), }) } From d196cc6faeae6339803a6422b2be35e7ef245dad Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 25 Jun 2024 15:18:44 +0200 Subject: [PATCH 53/56] Recreate rocksdb Signed-off-by: Danil --- .../crates/zk_inception/src/commands/external_node/init.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs index 09a2e634f2e2..7596d314f825 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs @@ -13,6 +13,7 @@ use crate::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_INITIALIZING_DATABASES_SPINNER, }, + utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; pub async fn run(shell: &Shell) -> anyhow::Result<()> { @@ -40,6 +41,11 @@ pub async fn init(shell: &Shell, chain_config: &ChainConfig) -> anyhow::Result<( .await .context(MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR)?; init_db(&db_config).await?; + recreate_rocksdb_dirs( + shell, + &chain_config.rocks_db_path, + RocksDBDirOption::ExternalNode, + )?; let path_to_server_migration = chain_config.link_to_code.join(SERVER_MIGRATIONS); migrate_db(shell, path_to_server_migration, &db_config.full_url()).await?; spin.finish(); From e4b9eac25a72a9a1aa66c308eb289dadb4d39936 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 25 Jun 2024 16:06:07 +0200 Subject: [PATCH 54/56] Refactor en Signed-off-by: Danil --- core/tests/ts-integration/src/env.ts | 1 - .../crates/zk_inception/src/commands/chain/genesis.rs | 6 +++--- .../crates/zk_inception/src/commands/external_node/init.rs | 6 +++--- zk_toolbox/crates/zk_inception/src/messages.rs | 4 +++- zk_toolbox/crates/zk_supervisor/Cargo.toml | 2 +- zk_toolbox/crates/zk_supervisor/src/dals.rs | 4 ++-- zk_toolbox/crates/zk_supervisor/src/messages.rs | 2 ++ 7 files changed, 14 insertions(+), 11 deletions(-) diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index bd8c40bbab60..ca97363fb4e2 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -367,7 +367,6 @@ function loadConfig(pathToHome: string, chainName: string, config: string, mode: suffixPath = path.join('external_node', suffixPath); } configPath = path.join(configPath, suffixPath); - console.log(configPath); if (!fs.existsSync(configPath)) { return []; } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs index 432b0045b3d3..554f9c2cf940 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs @@ -19,8 +19,8 @@ use crate::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, MSG_INITIALIZING_PROVER_DATABASE, - MSG_INITIALIZING_SERVER_DATABASE, MSG_SELECTED_CONFIG, MSG_STARTING_GENESIS, - MSG_STARTING_GENESIS_SPINNER, + MSG_INITIALIZING_SERVER_DATABASE, MSG_RECREATE_ROCKS_DB_ERRROR, MSG_SELECTED_CONFIG, + MSG_STARTING_GENESIS, MSG_STARTING_GENESIS_SPINNER, }, server::{RunServer, ServerMode}, utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, @@ -48,7 +48,7 @@ pub async fn genesis( shell.create_dir(&config.rocks_db_path)?; let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) - .context("Failed to create rocks db path")?; + .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; let mut general = config.get_general_config()?; general.set_rocks_db_config(rocks_db)?; if config.prover_version != ProverMode::NoProofs { diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs index 7596d314f825..c6101e88739c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/init.rs @@ -10,8 +10,8 @@ use xshell::Shell; use crate::{ consts::SERVER_MIGRATIONS, messages::{ - MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, - MSG_INITIALIZING_DATABASES_SPINNER, + MSG_CHAIN_NOT_INITIALIZED, MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED, + MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_INITIALIZING_DATABASES_SPINNER, }, utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; @@ -34,7 +34,7 @@ pub async fn init(shell: &Shell, chain_config: &ChainConfig) -> anyhow::Result<( chain_config .external_node_config_path .clone() - .context("External node is not initalized")?, + .context(MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED)?, )?; let db_config = DatabaseConfig::from_url(secrets.database.server_url)?; drop_db_if_exists(&db_config) diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 816e1f96be1d..1fa36fbabb1b 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -45,7 +45,6 @@ pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT: &str = "Provide the path t pub(super) const MSG_L1_RPC_URL_INVALID_ERR: &str = "Invalid RPC URL"; pub(super) const MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR: &str = "Invalid path"; pub(super) const MSG_GENESIS_DATABASE_ERR: &str = "Unable to perform genesis on the database"; -pub(super) const MSG_CONTRACTS_CONFIG_NOT_FOUND_ERR: &str = "Ecosystem contracts config not found"; pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; pub(super) const MSG_INITIALIZING_ECOSYSTEM: &str = "Initializing ecosystem"; pub(super) const MSG_DEPLOYING_ERC20: &str = "Deploying ERC20 contracts"; @@ -57,6 +56,7 @@ pub(super) const MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER: &str = "Deploying ecosystem contracts..."; pub(super) const MSG_REGISTERING_CHAIN_SPINNER: &str = "Registering chain..."; pub(super) const MSG_ACCEPTING_ADMIN_SPINNER: &str = "Accepting admin..."; +pub(super) const MSG_RECREATE_ROCKS_DB_ERRROR: &str = "Failed to create rocks db path"; pub(super) fn msg_initializing_chain(chain_name: &str) -> String { format!("Initializing chain {chain_name}") @@ -204,6 +204,8 @@ pub(super) fn msg_preparing_en_config_is_done(path: &Path) -> String { format!("External nodes configs could be found in: {path:?}") } +pub(super) const MSG_EXTERNAL_NODE_CONFIG_NOT_INITIALIZED: &str = + "External node is not initialized"; /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; diff --git a/zk_toolbox/crates/zk_supervisor/Cargo.toml b/zk_toolbox/crates/zk_supervisor/Cargo.toml index bb2b37aa6f75..d8f5d7862a04 100644 --- a/zk_toolbox/crates/zk_supervisor/Cargo.toml +++ b/zk_toolbox/crates/zk_supervisor/Cargo.toml @@ -21,4 +21,4 @@ strum_macros.workspace = true tokio.workspace = true url.workspace = true xshell.workspace = true -serde = { version = "1.0.197", features = ["derive"] } +serde.workspace = true diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index 90a8da3ec23c..ae8815c96899 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -4,7 +4,7 @@ use config::{EcosystemConfig, SecretsConfig}; use url::Url; use xshell::Shell; -use crate::messages::MSG_CHAIN_NOT_FOUND_ERR; +use crate::messages::{MSG_CHAIN_NOT_FOUND_ERR, MSG_PROVER_URL_MUST_BE_PRESENTED}; const CORE_DAL_PATH: &str = "core/lib/dal"; const PROVER_DAL_PATH: &str = "prover/prover_dal"; @@ -49,7 +49,7 @@ pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { url: secrets .database .prover_url - .context("Prover url must be presented")? + .context(MSG_PROVER_URL_MUST_BE_PRESENTED)? .clone(), }) } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 39b2920b8c8f..7ef956b8f545 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -37,6 +37,8 @@ pub(super) const MSG_DATABASE_RESET_PAST: &str = "reset"; pub(super) const MSG_DATABASE_SETUP_GERUND: &str = "Setting up"; pub(super) const MSG_DATABASE_SETUP_PAST: &str = "set up"; +pub(super) const MSG_PROVER_URL_MUST_BE_PRESENTED: &str = "Prover url must be presented"; + pub(super) const MSG_DATABASE_COMMON_PROVER_HELP: &str = "Prover database"; pub(super) const MSG_DATABASE_COMMON_CORE_HELP: &str = "Core database"; pub(super) const MSG_DATABASE_NEW_MIGRATION_DATABASE_HELP: &str = From 9c3d17da453083e7802826aaa45932628086127b Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 26 Jun 2024 11:00:25 +0200 Subject: [PATCH 55/56] Fix formatting Signed-off-by: Danil --- .github/workflows/ci-zk-toolbox-reusable.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 76e2e2e317cb..d00856a17c73 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -11,7 +11,7 @@ jobs: uses: ./.github/workflows/ci-core-lint-reusable.yml build: - runs-on: [ matterlabs-ci-runner ] + runs-on: [matterlabs-ci-runner] steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 @@ -48,8 +48,8 @@ jobs: compression-level: 0 integration_test: - runs-on: [ matterlabs-ci-runner ] - needs: [ build ] + runs-on: [matterlabs-ci-runner] + needs: [build] steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 From 84e77eb871f01ecd5677f32cdcdd09ce2da71b83 Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 26 Jun 2024 11:00:25 +0200 Subject: [PATCH 56/56] Fix formatting Signed-off-by: Danil --- .github/workflows/ci-zk-toolbox-reusable.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index d00856a17c73..83ec7d1f5dc3 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -101,16 +101,15 @@ jobs: - name: Run external node server run: | ci_run zk_inception external-node configs --db-url=postgres://postgres:notsecurepassword@postgres:5432 \ - --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 - ci_run zk_inception external-node init --ignore-prerequisites - ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log & + --db-name=zksync_en_localhost_era --l1-rpc-url=http://reth:8545 + ci_run zk_inception external-node init --ignore-prerequisites + ci_run zk_inception external-node run --ignore-prerequisites &>external_node.log & ci_run sleep 5 - name: Run integration tests en run: | ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose --external-node - - name: Show server.log logs if: always() run: ci_run cat server.log || true