From 4fdac8208b990118cbed8c1b7000583e44bb4f83 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 30 May 2024 12:12:01 +0200 Subject: [PATCH 01/35] File based config for en Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 + core/lib/config/src/configs/en_config.rs | 10 +++++++ core/lib/config/src/configs/mod.rs | 1 + core/lib/protobuf_config/src/en.rs | 29 +++++++++++++++++++ core/lib/protobuf_config/src/lib.rs | 1 + .../protobuf_config/src/proto/config/en.proto | 10 +++++++ core/lib/protobuf_config/src/tests.rs | 1 + etc/env/file_based/external_node.yaml | 3 ++ 8 files changed, 56 insertions(+) create mode 100644 core/lib/config/src/configs/en_config.rs create mode 100644 core/lib/protobuf_config/src/en.rs create mode 100644 core/lib/protobuf_config/src/proto/config/en.proto create mode 100644 etc/env/file_based/external_node.yaml diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 56d66a3a4253..b00e78a93b3c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -2,6 +2,7 @@ use std::{ env, ffi::OsString, num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + path::PathBuf, time::Duration, }; diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs new file mode 100644 index 000000000000..287a47ce4eaa --- /dev/null +++ b/core/lib/config/src/configs/en_config.rs @@ -0,0 +1,10 @@ +use serde::Deserialize; +use zksync_basic_types::{L1ChainId, L2ChainId}; + +/// Temporary config for initializing external node, will be completely replaced by consensus config later +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct ENConfig { + pub l2_chain_id: L2ChainId, + pub l1_chain_id: L1ChainId, + pub main_node_url: String, +} diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 925c30976f97..148a7b2c61a0 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -28,6 +28,7 @@ pub mod consensus; pub mod contract_verifier; pub mod contracts; pub mod database; +pub mod en_config; pub mod eth_sender; pub mod eth_watch; mod experimental; diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs new file mode 100644 index 000000000000..afc1a50e3723 --- /dev/null +++ b/core/lib/protobuf_config/src/en.rs @@ -0,0 +1,29 @@ +use zksync_basic_types::{L1ChainId, L2ChainId}; +use zksync_config::configs::en_config::ENConfig; +use zksync_protobuf::{required, ProtoRepr}; + +use crate::proto::en as proto; + +impl ProtoRepr for proto::ExternalNode { + type Type = ENConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + main_node_url: required(&self.main_node_url)?.to_string(), + l1_chain_id: required(&self.l1_chain_id) + .map(|x| L1ChainId(*x)) + .context("l1_chain_id")?, + l2_chain_id: required(&self.l2_chain_id) + .and_then(|x| L2ChainId::try_from(*x).map_err(|a| anyhow::anyhow!(a))) + .context("l2_chain_id")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + main_node_url: Some(this.main_node_url.clone()), + l1_chain_id: Some(this.l1_chain_id.0), + l2_chain_id: Some(this.l2_chain_id.as_u64()), + } + } +} diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index 25d5662b9ddb..6eca9be9c2e3 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -11,6 +11,7 @@ mod consensus; mod contract_verifier; mod contracts; mod database; +mod en; mod eth; mod experimental; mod general; diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto new file mode 100644 index 000000000000..ed2553f5aca5 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.en; + + +message ExternalNode { + optional string main_node_url = 1; // required + optional uint64 l2_chain_id = 2; // required + optional uint64 l1_chain_id = 3; // required +} diff --git a/core/lib/protobuf_config/src/tests.rs b/core/lib/protobuf_config/src/tests.rs index fad37700ae5f..f5a36b2f4e30 100644 --- a/core/lib/protobuf_config/src/tests.rs +++ b/core/lib/protobuf_config/src/tests.rs @@ -65,4 +65,5 @@ fn verify_file_parsing() { decode_yaml_repr::(&base_path.join("contracts.yaml"), true) .unwrap(); decode_yaml_repr::(&base_path.join("secrets.yaml"), true).unwrap(); + decode_yaml_repr::(&base_path.join("secrets.yaml"), true).unwrap(); } diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml new file mode 100644 index 000000000000..a909fdfa9ba0 --- /dev/null +++ b/etc/env/file_based/external_node.yaml @@ -0,0 +1,3 @@ +l1_chain_id: 9 +l2_chain_id: 270 +main_node_url: http://localhost:3050 \ No newline at end of file From d232f27f831320074de4b36981c819bd88c717b7 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 30 May 2024 19:24:28 +0200 Subject: [PATCH 02/35] Implement external node config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 259 +++++++++++++++++- .../external_node/src/config/observability.rs | 29 ++ core/lib/config/src/configs/api.rs | 4 + core/lib/config/src/configs/en_config.rs | 28 +- core/lib/config/src/configs/experimental.rs | 6 + core/lib/config/src/testonly.rs | 5 + core/lib/protobuf_config/src/api.rs | 10 +- core/lib/protobuf_config/src/en.rs | 88 +++++- core/lib/protobuf_config/src/experimental.rs | 10 + core/lib/protobuf_config/src/genesis.rs | 5 +- .../src/proto/config/api.proto | 3 +- .../protobuf_config/src/proto/config/en.proto | 18 ++ .../src/proto/config/experimental.proto | 3 + .../src/temp_config_store/mod.rs | 9 + etc/env/file_based/external_node.yaml | 3 +- etc/env/file_based/general.yaml | 2 +- 16 files changed, 468 insertions(+), 14 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 56d66a3a4253..150c9c4cee5f 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -2,19 +2,23 @@ use std::{ env, ffi::OsString, num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + path::PathBuf, time::Duration, }; use anyhow::Context; +use futures::StreamExt; use serde::Deserialize; use zksync_config::{ configs::{ api::{MaxResponseSize, MaxResponseSizeOverrides}, consensus::{ConsensusConfig, ConsensusSecrets}, + en_config::ENConfig, + GeneralConfig, Secrets, }, ObjectStoreConfig, }; -use zksync_core_leftovers::temp_config_store::decode_yaml_repr; +use zksync_core_leftovers::temp_config_store::{decode_yaml_repr, read_yaml_repr}; #[cfg(test)] use zksync_dal::{ConnectionPool, Core}; use zksync_metadata_calculator::MetadataCalculatorRecoveryConfig; @@ -415,6 +419,134 @@ pub(crate) struct OptionalENConfig { } impl OptionalENConfig { + fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { + let api = general_config + .api_config + .as_ref() + .context("Api is required")?; + let db = general_config + .db_config + .as_ref() + .context("Db config is required")?; + let postgres = general_config + .postgres_config + .as_ref() + .context("Db config is required")?; + let state_keeper = general_config + .state_keeper_config + .as_ref() + .context("Db config is required")?; + + let api_namespaces = api + .web3_json_rpc + .api_namespaces + .as_ref() + .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + + Ok(OptionalENConfig { + filters_limit: api + .web3_json_rpc + .filters_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_filters_limit), + subscriptions_limit: api + .web3_json_rpc + .subscriptions_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_subscriptions_limit), + req_entities_limit: api + .web3_json_rpc + .req_entities_limit + .map(|a| a as usize) + .unwrap_or_else(OptionalENConfig::default_req_entities_limit), + max_tx_size_bytes: api.web3_json_rpc.max_tx_size, + vm_execution_cache_misses_limit: api.web3_json_rpc.vm_execution_cache_misses_limit, + transactions_per_sec_limit: None, + fee_history_limit: api.web3_json_rpc.fee_history_limit(), + max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), + max_response_body_size_mb: api.web3_json_rpc.max_response_body_size().global, + max_response_body_size_overrides_mb: api + .web3_json_rpc + .max_response_body_size() + .overrides, + pubsub_polling_interval_ms: api.web3_json_rpc.pubsub_interval().as_millis() as u64, + max_nonce_ahead: api.web3_json_rpc.max_nonce_ahead, + vm_concurrency_limit: api.web3_json_rpc.vm_concurrency_limit(), + factory_deps_cache_size_mb: api.web3_json_rpc.factory_deps_cache_size(), + initial_writes_cache_size_mb: api.web3_json_rpc.initial_writes_cache_size(), + latest_values_cache_size_mb: api.web3_json_rpc.latest_values_cache_size(), + filters_disabled: api.web3_json_rpc.filters_disabled, + mempool_cache_update_interval_ms: api + .web3_json_rpc + .mempool_cache_update_interval() + .as_millis() as u64, + mempool_cache_size: api.web3_json_rpc.mempool_cache_size(), + + healthcheck_slow_time_limit_ms: api.healthcheck.slow_time_limit_ms, + healthcheck_hard_time_limit_ms: api.healthcheck.hard_time_limit_ms, + estimate_gas_scale_factor: api.web3_json_rpc.gas_price_scale_factor, + estimate_gas_acceptable_overestimation: api + .web3_json_rpc + .estimate_gas_acceptable_overestimation, + gas_price_scale_factor: api.web3_json_rpc.gas_price_scale_factor, + merkle_tree_max_l1_batches_per_iter: db.merkle_tree.max_l1_batches_per_iter, + merkle_tree_max_open_files: db.experimental.state_keeper_db_max_open_files, + merkle_tree_multi_get_chunk_size: db.merkle_tree.multi_get_chunk_size, + merkle_tree_block_cache_size_mb: db + .experimental + .state_keeper_db_block_cache_capacity_mb, + merkle_tree_memtable_capacity_mb: db.merkle_tree.memtable_capacity_mb, + merkle_tree_stalled_writes_timeout_sec: db.merkle_tree.stalled_writes_timeout_sec, + database_long_connection_threshold_ms: postgres.long_connection_threshold_ms, + database_slow_query_threshold_ms: postgres.slow_query_threshold_ms, + l2_block_seal_queue_capacity: state_keeper.l2_block_seal_queue_capacity, + main_node_rate_limit_rps: Self::default_main_node_rate_limit_rps(), + l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, + snapshots_recovery_enabled: enconfig + .snapshot_recovery + .as_ref() + .map(|a| a.enabled) + .unwrap_or_default(), + snapshots_recovery_postgres_max_concurrency: enconfig + .snapshot_recovery + .as_ref() + .map(|a| a.postgres_max_concurrency) + .flatten() + .unwrap_or_else(Self::default_snapshots_recovery_postgres_max_concurrency), + pruning_enabled: enconfig + .pruning + .as_ref() + .map(|a| a.enabled) + .unwrap_or_default(), + pruning_chunk_size: enconfig + .pruning + .as_ref() + .map(|a| a.chunk_size) + .flatten() + .unwrap_or_else(Self::default_pruning_chunk_size), + pruning_removal_delay_sec: enconfig + .pruning + .as_ref() + .map(|a| a.removal_delay_sec) + .flatten() + .unwrap_or_else(Self::default_pruning_removal_delay_sec), + pruning_data_retention_sec: enconfig + .pruning + .as_ref() + .map(|a| a.data_retention_sec) + .flatten() + .unwrap_or_else(Self::default_pruning_data_retention_sec), + contracts_diamond_proxy_addr: None, + protective_reads_persistence_enabled: db.experimental.reads_persistence_enabled, + merkle_tree_processing_delay_ms: db.experimental.processing_delay_ms, + merkle_tree_include_indices_and_filters_in_block_cache: db + .experimental + .include_indices_and_filters_in_block_cache, + api_namespaces, + extended_rpc_tracing: api.web3_json_rpc.extended_api_tracing, + }) + } + const fn default_filters_limit() -> usize { 10_000 } @@ -676,6 +808,34 @@ impl RequiredENConfig { .context("could not load external node config") } + fn from_configs( + general: &GeneralConfig, + en_config: &ENConfig, + secrets: &Secrets, + ) -> anyhow::Result { + let api_config = general + .api_config + .as_ref() + .context("Api config is required")?; + let db_config = general.db_config.as_ref().context("Database config")?; + Ok(RequiredENConfig { + l1_chain_id: en_config.l1_chain_id, + l2_chain_id: en_config.l2_chain_id, + http_port: api_config.web3_json_rpc.http_port, + ws_port: api_config.web3_json_rpc.ws_port, + healthcheck_port: api_config.healthcheck.port, + eth_client_url: secrets + .l1 + .as_ref() + .context("Eth config is required")? + .l1_rpc_url + .clone(), + main_node_url: en_config.main_node_url.clone(), + state_cache_path: db_config.state_keeper_db_path.clone(), + merkle_tree_path: db_config.merkle_tree.path.clone(), + }) + } + #[cfg(test)] fn mock(temp_dir: &tempfile::TempDir) -> Self { Self { @@ -786,6 +946,26 @@ impl ExperimentalENConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } + pub fn from_configs( + general_config: &GeneralConfig, + enconfig: &ENConfig, + ) -> anyhow::Result { + let db_config = general_config.db_config.as_ref().context("db_config")?; + let snapshot_recovery = enconfig + .snapshot_recovery + .as_ref() + .context("snapshot_recovery")?; + Ok(Self { + state_keeper_db_block_cache_capacity_mb: db_config + .experimental + .state_keeper_db_block_cache_capacity_mb, + state_keeper_db_max_open_files: db_config.experimental.state_keeper_db_max_open_files, + snapshots_recovery_tree_chunk_size: snapshot_recovery + .tree_chunk_size + .unwrap_or_else(Self::default_snapshots_recovery_tree_chunk_size), + commitment_generator_max_parallelism: enconfig.commitment_generator_max_parallelism, + }) + } } pub(crate) fn read_consensus_secrets() -> anyhow::Result> { @@ -834,11 +1014,29 @@ pub struct ApiComponentConfig { pub tree_api_remote_url: Option, } +impl ApiComponentConfig { + fn from_configs(en_config: &ENConfig) -> Self { + ApiComponentConfig { + tree_api_remote_url: en_config.tree_api_remote_url.clone(), + } + } +} + #[derive(Debug, Deserialize)] pub struct TreeComponentConfig { pub api_port: Option, } +impl TreeComponentConfig { + fn from_configs(general_config: &GeneralConfig) -> Self { + let api_port = general_config + .api_config + .as_ref() + .map(|a| a.merkle_tree.port); + TreeComponentConfig { api_port } + } +} + /// External Node Config contains all the configuration required for the EN operation. /// It is split into three parts: required, optional and remote for easier navigation. #[derive(Debug)] @@ -876,6 +1074,65 @@ impl ExternalNodeConfig<()> { }) } + pub fn from_files( + general_config_path: PathBuf, + external_node_config_patch: PathBuf, + secrets_configs: PathBuf, + consensus: Option, + ) -> anyhow::Result { + let general_config = read_yaml_repr::(general_config_path) + .context("failed decoding general YAML config")?; + let external_node_config = + read_yaml_repr::(external_node_config_patch) + .context("failed decoding external node YAML config")?; + let secrets_config = read_yaml_repr::(secrets_configs) + .context("failed decoding secrets YAML config")?; + + let consensus = consensus + .map(|path| read_yaml_repr::(path)) + .transpose() + .context("failed decoding consensus YAML config")?; + + let required = RequiredENConfig::from_configs( + &general_config, + &external_node_config, + &secrets_config, + )?; + let optional = OptionalENConfig::from_configs(&general_config, &external_node_config)?; + let postgres = PostgresConfig { + database_url: secrets_config + .database + .as_ref() + .context("DB secrets is required")? + .server_url + .clone() + .context("Server url is required")?, + max_connections: general_config + .postgres_config + .as_ref() + .context("Postgres config is required")? + .max_connections()?, + }; + let observability = ObservabilityENConfig::from_configs(&general_config)?; + let experimental = + ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; + + let api_component = ApiComponentConfig::from_configs(&external_node_config); + let tree_component = TreeComponentConfig::from_configs(&general_config); + + Ok(Self { + required, + postgres, + optional, + observability, + experimental, + consensus, + api_component, + tree_component, + remote: (), + }) + } + /// Fetches contracts addresses from the main node, completing the configuration. pub async fn fetch_remote( self, diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index a571b071b5e2..eec8bf2d76a7 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use serde::Deserialize; use vlog::LogFormat; +use zksync_config::configs::{GeneralConfig, ObservabilityConfig}; use super::{ConfigurationSource, Environment}; @@ -98,4 +99,32 @@ impl ObservabilityENConfig { } Ok(guard) } + + pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { + let observability = general_config + .observability + .as_ref() + .context("Observability is required")?; + let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = + if let Some(api) = general_config.api_config.as_ref() { + ( + Some(api.prometheus.listener_port), + Some(api.prometheus.pushgateway_url.clone()), + api.prometheus.push_interval_ms.unwrap_or_default(), + ) + } else { + (None, None, 0) + }; + Ok(Self { + prometheus_port, + prometheus_pushgateway_url, + prometheus_push_interval_ms, + sentry_url: observability.sentry_url.clone(), + sentry_environment: observability.sentry_environment.clone(), + log_format: observability + .log_format + .parse() + .context("Invalid log format")?, + }) + } } diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index 3b33ef43343f..e28d61fa055b 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,6 +213,8 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + pub api_namespaces: Option>, + pub extended_api_tracing: bool, } impl Web3JsonRpcConfig { @@ -251,6 +253,8 @@ impl Web3JsonRpcConfig { mempool_cache_size: Default::default(), tree_api_url: None, whitelisted_tokens_for_aa: Default::default(), + api_namespaces: None, + extended_api_tracing: false, } } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 287a47ce4eaa..a07a0c8972ab 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,10 +1,34 @@ +use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; + use serde::Deserialize; -use zksync_basic_types::{L1ChainId, L2ChainId}; +use zksync_basic_types::{ + commitment::L1BatchCommitmentMode, url::SensitiveUrl, L1ChainId, L2ChainId, +}; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct Pruning { + pub enabled: bool, + pub chunk_size: Option, + pub removal_delay_sec: Option, + pub data_retention_sec: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct SnapshotRecovery { + pub enabled: bool, + pub postgres_max_concurrency: Option, + pub tree_chunk_size: Option, +} /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub main_node_url: String, + pub main_node_url: SensitiveUrl, + pub commitment_generator_max_parallelism: Option, + pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + pub tree_api_remote_url: Option, + pub snapshot_recovery: Option, + pub pruning: Option, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index ad0ef5a4d5b8..1fb10edc0826 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,6 +12,9 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + pub reads_persistence_enabled: bool, + pub processing_delay_ms: u64, + pub include_indices_and_filters_in_block_cache: bool, } impl Default for ExperimentalDBConfig { @@ -20,6 +23,9 @@ impl Default for ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, + reads_persistence_enabled: true, + processing_delay_ms: 0, + include_indices_and_filters_in_block_cache: false, } } } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 55e4d1c82767..b9ed5f272afc 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -97,6 +97,8 @@ impl Distribution for EncodeDist { mempool_cache_update_interval: self.sample(rng), mempool_cache_size: self.sample(rng), whitelisted_tokens_for_aa: self.sample_range(rng).map(|_| rng.gen()).collect(), + api_namespaces: self.sample_range(rng).map(|_| self.sample(rng)).collect(), + extended_api_tracing: self.sample(rng), } } } @@ -280,6 +282,9 @@ impl Distribution for EncodeDist { configs::ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: self.sample(rng), state_keeper_db_max_open_files: self.sample(rng), + reads_persistence_enabled: self.sample(rng), + processing_delay_ms: self.sample(rng), + include_indices_and_filters_in_block_cache: self.sample(rng), } } } diff --git a/core/lib/protobuf_config/src/api.rs b/core/lib/protobuf_config/src/api.rs index fe0cfb3e0d6e..4eac849773f3 100644 --- a/core/lib/protobuf_config/src/api.rs +++ b/core/lib/protobuf_config/src/api.rs @@ -69,7 +69,11 @@ impl ProtoRepr for proto::Web3JsonRpc { }) .collect::>() .context("max_response_body_size_overrides")?; - + let api_namespaces = if self.api_namespaces.is_empty() { + None + } else { + Some(self.api_namespaces.clone()) + }; Ok(Self::Type { http_port: required(&self.http_port) .and_then(|p| Ok((*p).try_into()?)) @@ -154,6 +158,8 @@ impl ProtoRepr for proto::Web3JsonRpc { .map(|(i, k)| parse_h160(k).context(i)) .collect::, _>>() .context("account_pks")?, + extended_api_tracing: self.extended_api_tracing.unwrap_or_default(), + api_namespaces, }) } @@ -222,6 +228,8 @@ impl ProtoRepr for proto::Web3JsonRpc { .iter() .map(|k| format!("{:?}", k)) .collect(), + extended_api_tracing: Some(this.extended_api_tracing), + api_namespaces: this.api_namespaces.clone().unwrap_or_default(), } } } diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 3e90feceb11e..576934aeddfa 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,30 +1,108 @@ +use std::{ + num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + str::FromStr, +}; + use anyhow::Context; -use zksync_basic_types::{L1ChainId, L2ChainId}; -use zksync_config::configs::en_config::ENConfig; +use zksync_basic_types::{url::SensitiveUrl, L1ChainId, L2ChainId}; +use zksync_config::configs::en_config::{ENConfig, Pruning, SnapshotRecovery}; use zksync_protobuf::{required, ProtoRepr}; -use crate::proto::en as proto; +use crate::{proto::en as proto, read_optional_repr}; + +impl ProtoRepr for proto::Pruning { + type Type = Pruning; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + chunk_size: self.chunk_size, + removal_delay_sec: self.removal_delay_sec.map(|a| NonZeroU64::new(a)).flatten(), + data_retention_sec: self.data_retention_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + chunk_size: this.chunk_size, + removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), + data_retention_sec: this.data_retention_sec, + } + } +} + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecovery; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + postgres_max_concurrency: self + .postgres_max_concurrency + .map(|a| NonZeroUsize::new(a as usize)) + .flatten(), + tree_chunk_size: self.tree_chunk_size, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), + tree_chunk_size: this.tree_chunk_size, + } + } +} impl ProtoRepr for proto::ExternalNode { type Type = ENConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { - main_node_url: required(&self.main_node_url)?.to_string(), + main_node_url: SensitiveUrl::from_str( + required(&self.main_node_url).context("main_node_url")?, + )?, l1_chain_id: required(&self.l1_chain_id) .map(|x| L1ChainId(*x)) .context("l1_chain_id")?, l2_chain_id: required(&self.l2_chain_id) .and_then(|x| L2ChainId::try_from(*x).map_err(|a| anyhow::anyhow!(a))) .context("l2_chain_id")?, + l1_batch_commit_data_generator_mode: required( + &self.l1_batch_commit_data_generator_mode, + ) + .and_then(|x| Ok(crate::proto::genesis::L1BatchCommitDataGeneratorMode::try_from(*x)?)) + .context("l1_batch_commit_data_generator_mode")? + .parse(), + commitment_generator_max_parallelism: self + .commitment_generator_max_parallelism + .map(NonZeroU32::new) + .flatten(), + tree_api_remote_url: self.tree_api_remote_url.clone(), + pruning: read_optional_repr(&self.pruning).context("pruning")?, + snapshot_recovery: read_optional_repr(&self.snapshot_recovery) + .context("snapshot_recovery")?, }) } fn build(this: &Self::Type) -> Self { Self { - main_node_url: Some(this.main_node_url.clone()), + main_node_url: Some(this.main_node_url.expose_str().to_string()), l1_chain_id: Some(this.l1_chain_id.0), l2_chain_id: Some(this.l2_chain_id.as_u64()), + l1_batch_commit_data_generator_mode: Some( + crate::proto::genesis::L1BatchCommitDataGeneratorMode::new( + &this.l1_batch_commit_data_generator_mode, + ) + .into(), + ), + tree_api_remote_url: this.tree_api_remote_url.clone(), + commitment_generator_max_parallelism: this + .commitment_generator_max_parallelism + .map(|a| a.get()), + snapshot_recovery: None, + pruning: None, } } } diff --git a/core/lib/protobuf_config/src/experimental.rs b/core/lib/protobuf_config/src/experimental.rs index c4fe17aadf43..e3d864c51c42 100644 --- a/core/lib/protobuf_config/src/experimental.rs +++ b/core/lib/protobuf_config/src/experimental.rs @@ -21,6 +21,11 @@ impl ProtoRepr for proto::Db { .map(|count| NonZeroU32::new(count).context("cannot be 0")) .transpose() .context("state_keeper_db_max_open_files")?, + reads_persistence_enabled: self.reads_persistence_enabled.unwrap_or_default(), + processing_delay_ms: self.processing_delay_ms.unwrap_or_default(), + include_indices_and_filters_in_block_cache: self + .include_indices_and_filters_in_block_cache + .unwrap_or_default(), }) } @@ -34,6 +39,11 @@ impl ProtoRepr for proto::Db { state_keeper_db_max_open_files: this .state_keeper_db_max_open_files .map(NonZeroU32::get), + reads_persistence_enabled: Some(this.reads_persistence_enabled), + processing_delay_ms: Some(this.processing_delay_ms), + include_indices_and_filters_in_block_cache: Some( + this.include_indices_and_filters_in_block_cache, + ), } } } diff --git a/core/lib/protobuf_config/src/genesis.rs b/core/lib/protobuf_config/src/genesis.rs index 754f1fc16d0e..a6c23c653ad2 100644 --- a/core/lib/protobuf_config/src/genesis.rs +++ b/core/lib/protobuf_config/src/genesis.rs @@ -11,20 +11,21 @@ use zksync_protobuf::{repr::ProtoRepr, required}; use crate::{parse_h160, parse_h256, proto::genesis as proto}; impl proto::L1BatchCommitDataGeneratorMode { - fn new(n: &L1BatchCommitmentMode) -> Self { + pub(crate) fn new(n: &L1BatchCommitmentMode) -> Self { match n { L1BatchCommitmentMode::Rollup => Self::Rollup, L1BatchCommitmentMode::Validium => Self::Validium, } } - fn parse(&self) -> L1BatchCommitmentMode { + pub(crate) fn parse(&self) -> L1BatchCommitmentMode { match self { Self::Rollup => L1BatchCommitmentMode::Rollup, Self::Validium => L1BatchCommitmentMode::Validium, } } } + impl ProtoRepr for proto::Genesis { type Type = configs::GenesisConfig; fn read(&self) -> anyhow::Result { diff --git a/core/lib/protobuf_config/src/proto/config/api.proto b/core/lib/protobuf_config/src/proto/config/api.proto index 09503056a3f1..d7f0a5326476 100644 --- a/core/lib/protobuf_config/src/proto/config/api.proto +++ b/core/lib/protobuf_config/src/proto/config/api.proto @@ -40,7 +40,8 @@ message Web3JsonRpc { optional uint64 mempool_cache_size = 29; // optional repeated string whitelisted_tokens_for_aa = 30; // optional repeated MaxResponseSizeOverride max_response_body_size_overrides = 31; - + repeated string api_namespaces = 32; + optional bool extended_api_tracing = 33; reserved 15; reserved "l1_to_l2_transactions_compatibility_mode"; } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index ed2553f5aca5..5a4c47f8240d 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -1,10 +1,28 @@ syntax = "proto3"; +import "zksync/config/genesis.proto"; package zksync.config.en; +message SnapshotRecovery { + optional bool enabled = 1; + optional uint64 postgres_max_concurrency = 2; + optional uint64 tree_chunk_size = 3; +} + +message Pruning { + optional bool enabled = 1; + optional uint32 chunk_size = 2; + optional uint64 removal_delay_sec = 3; + optional uint64 data_retention_sec = 4; +} message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required + optional uint32 commitment_generator_max_parallelism = 4; + optional string tree_api_remote_url = 5; + optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 6; // optional, default to rollup + optional SnapshotRecovery snapshot_recovery = 7; + optional Pruning pruning = 8; } diff --git a/core/lib/protobuf_config/src/proto/config/experimental.proto b/core/lib/protobuf_config/src/proto/config/experimental.proto index 4f456b9aca39..ddcb4265319d 100644 --- a/core/lib/protobuf_config/src/proto/config/experimental.proto +++ b/core/lib/protobuf_config/src/proto/config/experimental.proto @@ -8,4 +8,7 @@ package zksync.config.experimental; message DB { optional uint64 state_keeper_db_block_cache_capacity_mb = 1; // MB; required optional uint32 state_keeper_db_max_open_files = 2; // optional + optional bool reads_persistence_enabled = 3; + optional uint64 processing_delay_ms = 4; + optional bool include_indices_and_filters_in_block_cache = 5; } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index cfac1df27cd0..6eecc35da95e 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -1,3 +1,6 @@ +use std::path::PathBuf; + +use anyhow::Context; use zksync_config::{ configs::{ api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, @@ -28,6 +31,12 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { let this: T = zksync_protobuf::serde::deserialize_proto_with_options(d, false)?; this.read() } + +pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { + let yaml = std::fs::read_to_string(&path_buf).context("failed decoding YAML config")?; + decode_yaml_repr::(&yaml) +} + // // TODO (QIT-22): This structure is going to be removed when components will be responsible for their own configs. /// A temporary config store allowing to pass deserialized configs from `zksync_server` to `zksync_core`. diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index a909fdfa9ba0..5a3ff7af5bd6 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,3 +1,4 @@ l1_chain_id: 9 l2_chain_id: 270 -main_node_url: http://localhost:3050 \ No newline at end of file +main_node_url: http://localhost:3050 +l1_batch_commit_data_generator_mode: Rollup \ No newline at end of file diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index d59da18d1266..2b7cfff81a2f 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -128,7 +128,7 @@ eth: aggregated_block_execute_deadline: 10 timestamp_criteria_max_allowed_lag: 30 max_eth_tx_data_size: 120000 - aggregated_proof_sizes: [ 1,4 ] + aggregated_proof_sizes: 1 max_aggregated_tx_gas: 4000000 max_acceptable_priority_fee_in_gwei: 100000000000 pubdata_sending_mode: BLOBS From d6f5535eed27aa67030ca759879f862a68e3f85a Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 10:48:30 +0200 Subject: [PATCH 03/35] Macros Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 67 ++++++++++++++++++------ 1 file changed, 50 insertions(+), 17 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 150c9c4cee5f..6af5bbe54a3b 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -45,6 +45,32 @@ pub(crate) mod observability; #[cfg(test)] mod tests; +macro_rules! load_optional_config_or_default { + ($config:expr, $($name:ident).+, $default:ident) => { + $config + .as_ref() + .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + .unwrap_or_else(OptionalENConfig::$default) + }; +} + +macro_rules! load_config_or_default { + ($config:expr, $($name:ident).+, $default:ident) => { + $config + .as_ref() + .map(|a| a.$($name).+.try_into()).transpose()? + .unwrap_or_else(OptionalENConfig::$default) + }; +} + +macro_rules! load_config { + ($config:expr, $($name:ident).+) => { + $config + .as_ref() + .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + }; +} + const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024; /// Encapsulation of configuration source with a mock implementation used in tests. @@ -444,23 +470,30 @@ impl OptionalENConfig { .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); Ok(OptionalENConfig { - filters_limit: api - .web3_json_rpc - .filters_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_filters_limit), - subscriptions_limit: api - .web3_json_rpc - .subscriptions_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_subscriptions_limit), - req_entities_limit: api - .web3_json_rpc - .req_entities_limit - .map(|a| a as usize) - .unwrap_or_else(OptionalENConfig::default_req_entities_limit), - max_tx_size_bytes: api.web3_json_rpc.max_tx_size, - vm_execution_cache_misses_limit: api.web3_json_rpc.vm_execution_cache_misses_limit, + filters_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.filters_limit, + default_filters_limit + ), + subscriptions_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.subscriptions_limit, + default_subscriptions_limit + ), + req_entities_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.req_entities_limit, + default_req_entities_limit + ), + max_tx_size_bytes: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_tx_size, + default_max_tx_size_bytes + ), + vm_execution_cache_misses_limit: load_config!( + general_config.api_config, + web3_json_rpc.vm_execution_cache_misses_limit + ), transactions_per_sec_limit: None, fee_history_limit: api.web3_json_rpc.fee_history_limit(), max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), From 8c035710832028d21836c73d918e40a539b79582 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 11:51:40 +0200 Subject: [PATCH 04/35] More defaults Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 76 +++++++++++++++++++----- 1 file changed, 61 insertions(+), 15 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 6af5bbe54a3b..2675f0a96a1c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -58,7 +58,7 @@ macro_rules! load_config_or_default { ($config:expr, $($name:ident).+, $default:ident) => { $config .as_ref() - .map(|a| a.$($name).+.try_into()).transpose()? + .map(|a| a.$($name).+.clone().try_into()).transpose()? .unwrap_or_else(OptionalENConfig::$default) }; } @@ -494,21 +494,63 @@ impl OptionalENConfig { general_config.api_config, web3_json_rpc.vm_execution_cache_misses_limit ), + // Option is deprecated transactions_per_sec_limit: None, - fee_history_limit: api.web3_json_rpc.fee_history_limit(), - max_batch_request_size: api.web3_json_rpc.max_batch_request_size(), - max_response_body_size_mb: api.web3_json_rpc.max_response_body_size().global, - max_response_body_size_overrides_mb: api - .web3_json_rpc - .max_response_body_size() - .overrides, - pubsub_polling_interval_ms: api.web3_json_rpc.pubsub_interval().as_millis() as u64, - max_nonce_ahead: api.web3_json_rpc.max_nonce_ahead, - vm_concurrency_limit: api.web3_json_rpc.vm_concurrency_limit(), - factory_deps_cache_size_mb: api.web3_json_rpc.factory_deps_cache_size(), - initial_writes_cache_size_mb: api.web3_json_rpc.initial_writes_cache_size(), - latest_values_cache_size_mb: api.web3_json_rpc.latest_values_cache_size(), - filters_disabled: api.web3_json_rpc.filters_disabled, + fee_history_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.fee_history_limit, + default_fee_history_limit + ), + max_batch_request_size: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.max_batch_request_size, + default_max_batch_request_size + ), + max_response_body_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.max_response_body_size_mb, + default_max_response_body_size_mb + ), + max_response_body_size_overrides_mb: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_response_body_size_overrides_mb, + default_max_response_body_size_overrides_mb + ), + pubsub_polling_interval_ms: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.pubsub_polling_interval, + default_polling_interval + ), + max_nonce_ahead: load_config_or_default!( + general_config.api_config, + web3_json_rpc.max_nonce_ahead, + default_max_nonce_ahead + ), + vm_concurrency_limit: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.vm_concurrency_limit, + default_vm_concurrency_limit + ), + factory_deps_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.factory_deps_cache_size_mb, + default_factory_deps_cache_size_mb + ), + initial_writes_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.initial_writes_cache_size_mb, + default_initial_writes_cache_size_mb + ), + latest_values_cache_size_mb: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.latest_values_cache_size_mb, + default_latest_values_cache_size_mb + ), + filters_disabled: general_config + .api_config + .as_ref() + .map(|a| a.web3_json_rpc.filters_disabled) + .unwrap_or_default(), mempool_cache_update_interval_ms: api .web3_json_rpc .mempool_cache_update_interval() @@ -671,6 +713,10 @@ impl OptionalENConfig { 10 } + fn default_max_response_body_size_overrides_mb() -> MaxResponseSizeOverrides { + MaxResponseSizeOverrides::empty() + } + const fn default_l2_block_seal_queue_capacity() -> usize { 10 } From c3ca813dd9a1a1ae47483df333263cc05df3b226 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 20:46:29 +0200 Subject: [PATCH 05/35] Use all optional en config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 234 +++++++++++------- core/lib/config/src/configs/en_config.rs | 3 +- core/lib/protobuf_config/src/en.rs | 9 +- .../protobuf_config/src/proto/config/en.proto | 7 +- 4 files changed, 154 insertions(+), 99 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 2675f0a96a1c..54ff79deba6c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -50,7 +50,7 @@ macro_rules! load_optional_config_or_default { $config .as_ref() .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? - .unwrap_or_else(OptionalENConfig::$default) + .unwrap_or_else(Self::$default) }; } @@ -59,7 +59,7 @@ macro_rules! load_config_or_default { $config .as_ref() .map(|a| a.$($name).+.clone().try_into()).transpose()? - .unwrap_or_else(OptionalENConfig::$default) + .unwrap_or_else(Self::$default) }; } @@ -67,7 +67,7 @@ macro_rules! load_config { ($config:expr, $($name:ident).+) => { $config .as_ref() - .map(|a| a.$($name).+.map(|a| a.try_into())).flatten().transpose()? + .map(|a| a.$($name).+.clone().map(|a| a.try_into())).flatten().transpose()? }; } @@ -446,28 +446,8 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { - let api = general_config - .api_config - .as_ref() - .context("Api is required")?; - let db = general_config - .db_config - .as_ref() - .context("Db config is required")?; - let postgres = general_config - .postgres_config - .as_ref() - .context("Db config is required")?; - let state_keeper = general_config - .state_keeper_config - .as_ref() - .context("Db config is required")?; - - let api_namespaces = api - .web3_json_rpc - .api_namespaces - .as_ref() - .map(|a| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) + .map(|a: Vec| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); Ok(OptionalENConfig { filters_limit: load_optional_config_or_default!( @@ -551,74 +531,138 @@ impl OptionalENConfig { .as_ref() .map(|a| a.web3_json_rpc.filters_disabled) .unwrap_or_default(), - mempool_cache_update_interval_ms: api - .web3_json_rpc - .mempool_cache_update_interval() - .as_millis() as u64, - mempool_cache_size: api.web3_json_rpc.mempool_cache_size(), - - healthcheck_slow_time_limit_ms: api.healthcheck.slow_time_limit_ms, - healthcheck_hard_time_limit_ms: api.healthcheck.hard_time_limit_ms, - estimate_gas_scale_factor: api.web3_json_rpc.gas_price_scale_factor, - estimate_gas_acceptable_overestimation: api - .web3_json_rpc - .estimate_gas_acceptable_overestimation, - gas_price_scale_factor: api.web3_json_rpc.gas_price_scale_factor, - merkle_tree_max_l1_batches_per_iter: db.merkle_tree.max_l1_batches_per_iter, - merkle_tree_max_open_files: db.experimental.state_keeper_db_max_open_files, - merkle_tree_multi_get_chunk_size: db.merkle_tree.multi_get_chunk_size, - merkle_tree_block_cache_size_mb: db - .experimental - .state_keeper_db_block_cache_capacity_mb, - merkle_tree_memtable_capacity_mb: db.merkle_tree.memtable_capacity_mb, - merkle_tree_stalled_writes_timeout_sec: db.merkle_tree.stalled_writes_timeout_sec, - database_long_connection_threshold_ms: postgres.long_connection_threshold_ms, - database_slow_query_threshold_ms: postgres.slow_query_threshold_ms, - l2_block_seal_queue_capacity: state_keeper.l2_block_seal_queue_capacity, - main_node_rate_limit_rps: Self::default_main_node_rate_limit_rps(), + mempool_cache_update_interval_ms: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.mempool_cache_update_interval, + default_mempool_cache_update_interval_ms + ), + mempool_cache_size: load_optional_config_or_default!( + general_config.api_config, + web3_json_rpc.mempool_cache_size, + default_mempool_cache_size + ), + + healthcheck_slow_time_limit_ms: load_config!( + general_config.api_config, + healthcheck.slow_time_limit_ms + ), + healthcheck_hard_time_limit_ms: load_config!( + general_config.api_config, + healthcheck.hard_time_limit_ms + ), + estimate_gas_scale_factor: load_config_or_default!( + general_config.api_config, + web3_json_rpc.estimate_gas_scale_factor, + default_estimate_gas_scale_factor + ), + estimate_gas_acceptable_overestimation: load_config_or_default!( + general_config.api_config, + web3_json_rpc.estimate_gas_acceptable_overestimation, + default_estimate_gas_acceptable_overestimation + ), + gas_price_scale_factor: load_config_or_default!( + general_config.api_config, + web3_json_rpc.gas_price_scale_factor, + default_gas_price_scale_factor + ), + merkle_tree_max_l1_batches_per_iter: load_config_or_default!( + general_config.db_config, + merkle_tree.max_l1_batches_per_iter, + default_merkle_tree_max_l1_batches_per_iter + ), + merkle_tree_max_open_files: load_config!( + general_config.db_config, + experimental.state_keeper_db_max_open_files + ), + merkle_tree_multi_get_chunk_size: load_config_or_default!( + general_config.db_config, + merkle_tree.multi_get_chunk_size, + default_merkle_tree_multi_get_chunk_size + ), + merkle_tree_block_cache_size_mb: load_config_or_default!( + general_config.db_config, + experimental.state_keeper_db_block_cache_capacity_mb, + default_merkle_tree_block_cache_size_mb + ), + merkle_tree_memtable_capacity_mb: load_config_or_default!( + general_config.db_config, + merkle_tree.memtable_capacity_mb, + default_merkle_tree_memtable_capacity_mb + ), + merkle_tree_stalled_writes_timeout_sec: load_config_or_default!( + general_config.db_config, + merkle_tree.stalled_writes_timeout_sec, + default_merkle_tree_stalled_writes_timeout_sec + ), + database_long_connection_threshold_ms: load_config!( + general_config.postgres_config, + long_connection_threshold_ms + ), + database_slow_query_threshold_ms: load_config!( + general_config.postgres_config, + slow_query_threshold_ms + ), + l2_block_seal_queue_capacity: load_config_or_default!( + general_config.state_keeper_config, + l2_block_seal_queue_capacity, + default_l2_block_seal_queue_capacity + ), l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, snapshots_recovery_enabled: enconfig .snapshot_recovery .as_ref() .map(|a| a.enabled) .unwrap_or_default(), - snapshots_recovery_postgres_max_concurrency: enconfig - .snapshot_recovery - .as_ref() - .map(|a| a.postgres_max_concurrency) - .flatten() - .unwrap_or_else(Self::default_snapshots_recovery_postgres_max_concurrency), + snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( + enconfig.snapshot_recovery, + postgres_max_concurrency, + default_snapshots_recovery_postgres_max_concurrency + ), pruning_enabled: enconfig .pruning .as_ref() .map(|a| a.enabled) .unwrap_or_default(), - pruning_chunk_size: enconfig - .pruning - .as_ref() - .map(|a| a.chunk_size) - .flatten() - .unwrap_or_else(Self::default_pruning_chunk_size), - pruning_removal_delay_sec: enconfig - .pruning + pruning_chunk_size: load_optional_config_or_default!( + enconfig.pruning, + chunk_size, + default_pruning_chunk_size + ), + pruning_removal_delay_sec: load_optional_config_or_default!( + enconfig.pruning, + removal_delay_sec, + default_pruning_removal_delay_sec + ), + pruning_data_retention_sec: load_optional_config_or_default!( + enconfig.pruning, + data_retention_sec, + default_pruning_data_retention_sec + ), + protective_reads_persistence_enabled: general_config + .db_config .as_ref() - .map(|a| a.removal_delay_sec) - .flatten() - .unwrap_or_else(Self::default_pruning_removal_delay_sec), - pruning_data_retention_sec: enconfig - .pruning + .map(|a| a.experimental.reads_persistence_enabled) + .unwrap_or_default(), + merkle_tree_processing_delay_ms: load_config_or_default!( + general_config.db_config, + experimental.processing_delay_ms, + default_merkle_tree_processing_delay_ms + ), + merkle_tree_include_indices_and_filters_in_block_cache: general_config + .db_config .as_ref() - .map(|a| a.data_retention_sec) - .flatten() - .unwrap_or_else(Self::default_pruning_data_retention_sec), - contracts_diamond_proxy_addr: None, - protective_reads_persistence_enabled: db.experimental.reads_persistence_enabled, - merkle_tree_processing_delay_ms: db.experimental.processing_delay_ms, - merkle_tree_include_indices_and_filters_in_block_cache: db - .experimental - .include_indices_and_filters_in_block_cache, + .map(|a| a.experimental.include_indices_and_filters_in_block_cache) + .unwrap_or_default(), + extended_rpc_tracing: load_config_or_default!( + general_config.api_config, + web3_json_rpc.extended_api_tracing, + default_extended_api_tracing + ), + main_node_rate_limit_rps: enconfig + .main_node_rate_limit_rps + .unwrap_or_else(Self::default_main_node_rate_limit_rps), api_namespaces, - extended_rpc_tracing: api.web3_json_rpc.extended_api_tracing, + contracts_diamond_proxy_addr: None, }) } @@ -1027,22 +1071,26 @@ impl ExperimentalENConfig { } pub fn from_configs( general_config: &GeneralConfig, - enconfig: &ENConfig, + external_node_config: &ENConfig, ) -> anyhow::Result { - let db_config = general_config.db_config.as_ref().context("db_config")?; - let snapshot_recovery = enconfig - .snapshot_recovery - .as_ref() - .context("snapshot_recovery")?; Ok(Self { - state_keeper_db_block_cache_capacity_mb: db_config - .experimental - .state_keeper_db_block_cache_capacity_mb, - state_keeper_db_max_open_files: db_config.experimental.state_keeper_db_max_open_files, - snapshots_recovery_tree_chunk_size: snapshot_recovery - .tree_chunk_size - .unwrap_or_else(Self::default_snapshots_recovery_tree_chunk_size), - commitment_generator_max_parallelism: enconfig.commitment_generator_max_parallelism, + state_keeper_db_block_cache_capacity_mb: load_config_or_default!( + general_config.db_config, + experimental.state_keeper_db_block_cache_capacity_mb, + default_state_keeper_db_block_cache_capacity_mb + ), + + state_keeper_db_max_open_files: load_config!( + general_config.db_config, + experimental.state_keeper_db_max_open_files + ), + snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( + external_node_config.snapshot_recovery, + tree_chunk_size, + default_snapshots_recovery_tree_chunk_size + ), + commitment_generator_max_parallelism: external_node_config + .commitment_generator_max_parallelism, }) } } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index a07a0c8972ab..20e8a9fc9558 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -26,8 +26,9 @@ pub struct ENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, pub main_node_url: SensitiveUrl, - pub commitment_generator_max_parallelism: Option, pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + pub main_node_rate_limit_rps: Option, + pub commitment_generator_max_parallelism: Option, pub tree_api_remote_url: Option, pub snapshot_recovery: Option, pub pruning: Option, diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 576934aeddfa..67a70a3ba7fb 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -80,6 +80,10 @@ impl ProtoRepr for proto::ExternalNode { .map(NonZeroU32::new) .flatten(), tree_api_remote_url: self.tree_api_remote_url.clone(), + main_node_rate_limit_rps: self + .main_node_rate_limit_rps + .map(|a| NonZeroUsize::new(a as usize)) + .flatten(), pruning: read_optional_repr(&self.pruning).context("pruning")?, snapshot_recovery: read_optional_repr(&self.snapshot_recovery) .context("snapshot_recovery")?, @@ -101,8 +105,9 @@ impl ProtoRepr for proto::ExternalNode { commitment_generator_max_parallelism: this .commitment_generator_max_parallelism .map(|a| a.get()), - snapshot_recovery: None, - pruning: None, + main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), + snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), + pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 5a4c47f8240d..45a4d68f12de 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -22,7 +22,8 @@ message ExternalNode { optional uint64 l1_chain_id = 3; // required optional uint32 commitment_generator_max_parallelism = 4; optional string tree_api_remote_url = 5; - optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 6; // optional, default to rollup - optional SnapshotRecovery snapshot_recovery = 7; - optional Pruning pruning = 8; + optional uint32 main_node_rate_limit_rps = 6; + optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup + optional SnapshotRecovery snapshot_recovery = 8; + optional Pruning pruning = 9; } From 78124cb4272fc5186e9539102c77fe8b5c405767 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 31 May 2024 21:11:52 +0200 Subject: [PATCH 06/35] Allowing to run external node from files Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 - .../external_node/src/config/observability.rs | 35 +++++++++++++------ core/bin/external_node/src/main.rs | 33 ++++++++++++++++- 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 54ff79deba6c..0b482fa84cea 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -7,7 +7,6 @@ use std::{ }; use anyhow::Context; -use futures::StreamExt; use serde::Deserialize; use zksync_config::{ configs::{ diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index eec8bf2d76a7..a4aa9970ca0d 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -4,7 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use serde::Deserialize; use vlog::LogFormat; -use zksync_config::configs::{GeneralConfig, ObservabilityConfig}; +use zksync_config::configs::GeneralConfig; use super::{ConfigurationSource, Environment}; @@ -27,6 +27,8 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, + /// Log directives, same as RUST_LOG + pub log_directives: Option, } impl ObservabilityENConfig { @@ -81,6 +83,9 @@ impl ObservabilityENConfig { pub fn build_observability(&self) -> anyhow::Result { let mut builder = vlog::ObservabilityBuilder::new().with_log_format(self.log_format); + if let Some(log_directives) = &self.log_directives { + builder = builder.with_log_directives(log_directives.clone()) + } // Some legacy deployments use `unset` as an equivalent of `None`. let sentry_url = self.sentry_url.as_deref().filter(|&url| url != "unset"); if let Some(sentry_url) = sentry_url { @@ -101,10 +106,20 @@ impl ObservabilityENConfig { } pub(crate) fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { - let observability = general_config - .observability - .as_ref() - .context("Observability is required")?; + let (sentry_url, sentry_environment, log_format, log_directives) = + if let Some(observability) = general_config.observability.as_ref() { + ( + observability.sentry_url.clone(), + observability.sentry_environment.clone(), + observability + .log_format + .parse() + .context("Invalid log format")?, + observability.log_directives.clone(), + ) + } else { + (None, None, LogFormat::default(), None) + }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = if let Some(api) = general_config.api_config.as_ref() { ( @@ -119,12 +134,10 @@ impl ObservabilityENConfig { prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms, - sentry_url: observability.sentry_url.clone(), - sentry_environment: observability.sentry_environment.clone(), - log_format: observability - .log_format - .parse() - .context("Invalid log format")?, + sentry_url, + sentry_environment, + log_format, + log_directives, }) } } diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 0f53e8983881..431cca1d6d70 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -707,6 +707,18 @@ struct Cli { /// Comma-separated list of components to launch. #[arg(long, default_value = "all")] components: ComponentsToRun, + /// Path to the yaml config. If set, it will be used instead of env vars. + #[arg(long)] + config_path: Option, + /// Path to the yaml with secrets. If set, it will be used instead of env vars. + #[arg(long)] + secrets_path: Option, + /// Path to the yaml with genesis. If set, it will be used instead of env vars. + #[arg(long)] + external_node_config_path: Option, + /// Path to the yaml with genesis. If set, it will be used instead of env vars. + #[arg(long)] + consensus_path: Option, } #[derive(Debug, Clone, Copy, PartialEq, Hash, Eq)] @@ -763,7 +775,26 @@ async fn main() -> anyhow::Result<()> { // Initial setup. let opt = Cli::parse(); - let mut config = ExternalNodeConfig::new().context("Failed to load node configuration")?; + let mut config = if let Some(config_path) = opt.config_path.clone() { + let secrets_path = opt + .secrets_path + .clone() + .context("File based config can't be used partially please provide all paths")?; + let external_node_config_path = opt + .external_node_config_path + .clone() + .context("File based config can't be used partially please provide all paths")?; + + ExternalNodeConfig::from_files( + config_path, + external_node_config_path, + secrets_path, + opt.consensus_path.clone(), + )? + } else { + ExternalNodeConfig::new().context("Failed to load node configuration")? + }; + if !opt.enable_consensus { config.consensus = None; } From da675b5757682c3048c9c27a4da4be439973a37a Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 12:30:21 +0200 Subject: [PATCH 07/35] Fix lints Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 2 +- core/bin/external_node/src/tests.rs | 8 ++++++++ core/lib/env_config/src/api.rs | 4 ++++ core/lib/protobuf_config/src/en.rs | 11 ++++------- .../src/temp_config_store/mod.rs | 2 +- 5 files changed, 18 insertions(+), 9 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 0b482fa84cea..e918c96ff50e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1215,7 +1215,7 @@ impl ExternalNodeConfig<()> { .context("failed decoding secrets YAML config")?; let consensus = consensus - .map(|path| read_yaml_repr::(path)) + .map(read_yaml_repr::) .transpose() .context("failed decoding consensus YAML config")?; diff --git a/core/bin/external_node/src/tests.rs b/core/bin/external_node/src/tests.rs index 00301e1b8234..5974e312d434 100644 --- a/core/bin/external_node/src/tests.rs +++ b/core/bin/external_node/src/tests.rs @@ -156,6 +156,10 @@ async fn external_node_basics(components_str: &'static str) { revert_pending_l1_batch: false, enable_consensus: false, components, + config_path: None, + secrets_path: None, + external_node_config_path: None, + consensus_path: None, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { @@ -265,6 +269,10 @@ async fn node_reacts_to_stop_signal_during_initial_reorg_detection() { revert_pending_l1_batch: false, enable_consensus: false, components: "core".parse().unwrap(), + config_path: None, + secrets_path: None, + external_node_config_path: None, + consensus_path: None, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { diff --git a/core/lib/env_config/src/api.rs b/core/lib/env_config/src/api.rs index 6f1948241c9e..68af37393bba 100644 --- a/core/lib/env_config/src/api.rs +++ b/core/lib/env_config/src/api.rs @@ -98,6 +98,8 @@ mod tests { addr("0x0000000000000000000000000000000000000001"), addr("0x0000000000000000000000000000000000000002"), ], + api_namespaces: Some(vec!["debug".to_string()]), + extended_api_tracing: true, }, prometheus: PrometheusConfig { listener_port: 3312, @@ -129,6 +131,8 @@ mod tests { API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 + API_WEB3_JSON_RPC_API_NAMESPACES=debug + API_WEB3_JSON_RPC_EXTENDED_API_TRACING=true API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" API_WEB3_JSON_RPC_WHITELISTED_TOKENS_FOR_AA="0x0000000000000000000000000000000000000001,0x0000000000000000000000000000000000000002" API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 67a70a3ba7fb..8e3b99d1aaa8 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -17,7 +17,7 @@ impl ProtoRepr for proto::Pruning { Ok(Self::Type { enabled: self.enabled.unwrap_or_default(), chunk_size: self.chunk_size, - removal_delay_sec: self.removal_delay_sec.map(|a| NonZeroU64::new(a)).flatten(), + removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), data_retention_sec: self.data_retention_sec, }) } @@ -40,8 +40,7 @@ impl ProtoRepr for proto::SnapshotRecovery { enabled: self.enabled.unwrap_or_default(), postgres_max_concurrency: self .postgres_max_concurrency - .map(|a| NonZeroUsize::new(a as usize)) - .flatten(), + .and_then(|a| NonZeroUsize::new(a as usize)), tree_chunk_size: self.tree_chunk_size, }) } @@ -77,13 +76,11 @@ impl ProtoRepr for proto::ExternalNode { .parse(), commitment_generator_max_parallelism: self .commitment_generator_max_parallelism - .map(NonZeroU32::new) - .flatten(), + .and_then(NonZeroU32::new), tree_api_remote_url: self.tree_api_remote_url.clone(), main_node_rate_limit_rps: self .main_node_rate_limit_rps - .map(|a| NonZeroUsize::new(a as usize)) - .flatten(), + .and_then(|a| NonZeroUsize::new(a as usize)), pruning: read_optional_repr(&self.pruning).context("pruning")?, snapshot_recovery: read_optional_repr(&self.snapshot_recovery) .context("snapshot_recovery")?, diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 6eecc35da95e..a2ea9cb8afe4 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -33,7 +33,7 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { } pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { - let yaml = std::fs::read_to_string(&path_buf).context("failed decoding YAML config")?; + let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; decode_yaml_repr::(&yaml) } From 3c17e1accf9c0bb6030b314f4c04752e829a7922 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:25:45 +0200 Subject: [PATCH 08/35] Add default for new envs Signed-off-by: Danil --- core/lib/config/src/configs/api.rs | 2 ++ core/lib/config/src/configs/experimental.rs | 11 ++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index e28d61fa055b..7616ee213cbc 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,7 +213,9 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + #[serde(default)] pub api_namespaces: Option>, + #[serde(default)] pub extended_api_tracing: bool, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 1fb10edc0826..833bc4556948 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,8 +12,11 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + #[serde(default = "ExperimentalDBConfig::default_protective_reads_persistence_enabled")] pub reads_persistence_enabled: bool, + #[serde(default = "ExperimentalDBConfig::default_merkle_tree_processing_delay_ms")] pub processing_delay_ms: u64, + #[serde(default)] pub include_indices_and_filters_in_block_cache: bool, } @@ -23,7 +26,7 @@ impl Default for ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, - reads_persistence_enabled: true, + reads_persistence_enabled: Self::default_protective_reads_persistence_enabled(), processing_delay_ms: 0, include_indices_and_filters_in_block_cache: false, } @@ -38,4 +41,10 @@ impl ExperimentalDBConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * super::BYTES_IN_MEGABYTE } + const fn default_protective_reads_persistence_enabled() -> bool { + true + } + const fn default_merkle_tree_processing_delay_ms() -> u64 { + 100 + } } From f7e65e4a6539d48e1435ef2f349c5fcac89710d0 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:42:49 +0200 Subject: [PATCH 09/35] Add more comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 1 + core/bin/external_node/src/main.rs | 2 +- core/lib/config/src/configs/api.rs | 3 +++ core/lib/config/src/configs/en_config.rs | 7 ++++++- core/lib/config/src/configs/experimental.rs | 9 +++++++++ core/lib/protobuf_config/src/proto/config/api.proto | 4 ++-- core/lib/protobuf_config/src/proto/config/en.proto | 6 +++--- 7 files changed, 25 insertions(+), 7 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e918c96ff50e..f5a15da2c616 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1068,6 +1068,7 @@ impl ExperimentalENConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } + pub fn from_configs( general_config: &GeneralConfig, external_node_config: &ENConfig, diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 431cca1d6d70..efa4d5da2209 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -716,7 +716,7 @@ struct Cli { /// Path to the yaml with genesis. If set, it will be used instead of env vars. #[arg(long)] external_node_config_path: Option, - /// Path to the yaml with genesis. If set, it will be used instead of env vars. + /// Path to the yaml with consensus. #[arg(long)] consensus_path: Option, } diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index 7616ee213cbc..e039ab10116a 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -213,8 +213,11 @@ pub struct Web3JsonRpcConfig { /// (additionally to natively bridged tokens). #[serde(default)] pub whitelisted_tokens_for_aa: Vec
, + /// Enabled JSON RPC API namespaces. If not set, all namespaces will be available #[serde(default)] pub api_namespaces: Option>, + /// Enables extended tracing of RPC calls. This may negatively impact performance for nodes under high load + /// (hundreds or thousands RPS). #[serde(default)] pub extended_api_tracing: bool, } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 20e8a9fc9558..ab8e49617e84 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -23,11 +23,16 @@ pub struct SnapshotRecovery { /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { + // Genesis pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub main_node_url: SensitiveUrl, pub l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + + // Main node configuration + pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, + + // En specific components pub commitment_generator_max_parallelism: Option, pub tree_api_remote_url: Option, pub snapshot_recovery: Option, diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 833bc4556948..31bdf3a4730f 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -12,10 +12,19 @@ pub struct ExperimentalDBConfig { /// Maximum number of files concurrently opened by state keeper cache RocksDB. Useful to fit into OS limits; can be used /// as a rudimentary way to control RAM usage of the cache. pub state_keeper_db_max_open_files: Option, + /// Configures whether to persist protective reads when persisting L1 batches in the state keeper. + /// Protective reads are never required by full nodes so far, not until such a node runs a full Merkle tree + /// (presumably, to participate in L1 batch proving). + /// By default, set to `true` as a temporary safety measure. #[serde(default = "ExperimentalDBConfig::default_protective_reads_persistence_enabled")] pub reads_persistence_enabled: bool, + // Merkle tree config + /// Processing delay between processing L1 batches in the Merkle tree. #[serde(default = "ExperimentalDBConfig::default_merkle_tree_processing_delay_ms")] pub processing_delay_ms: u64, + /// If specified, RocksDB indices and Bloom filters will be managed by the block cache, rather than + /// being loaded entirely into RAM on the RocksDB initialization. The block cache capacity should be increased + /// correspondingly; otherwise, RocksDB performance can significantly degrade. #[serde(default)] pub include_indices_and_filters_in_block_cache: bool, } diff --git a/core/lib/protobuf_config/src/proto/config/api.proto b/core/lib/protobuf_config/src/proto/config/api.proto index d7f0a5326476..4fea0691f79d 100644 --- a/core/lib/protobuf_config/src/proto/config/api.proto +++ b/core/lib/protobuf_config/src/proto/config/api.proto @@ -40,8 +40,8 @@ message Web3JsonRpc { optional uint64 mempool_cache_size = 29; // optional repeated string whitelisted_tokens_for_aa = 30; // optional repeated MaxResponseSizeOverride max_response_body_size_overrides = 31; - repeated string api_namespaces = 32; - optional bool extended_api_tracing = 33; + repeated string api_namespaces = 32; // Optional, if empty all namespaces are available + optional bool extended_api_tracing = 33; // optional, default false reserved 15; reserved "l1_to_l2_transactions_compatibility_mode"; } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 45a4d68f12de..375872d36df4 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -20,9 +20,9 @@ message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required - optional uint32 commitment_generator_max_parallelism = 4; - optional string tree_api_remote_url = 5; - optional uint32 main_node_rate_limit_rps = 6; + optional uint32 commitment_generator_max_parallelism = 4; // optional + optional string tree_api_remote_url = 5; // optional + optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup optional SnapshotRecovery snapshot_recovery = 8; optional Pruning pruning = 9; From 3e5d6c7bfbca0321fa7aa6cccf7597541d1c3d86 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 13:42:49 +0200 Subject: [PATCH 10/35] Add more comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 8 ++++++-- core/bin/zksync_server/src/node_builder.rs | 8 +++++++- etc/env/file_based/external_node.yaml | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index f5a15da2c616..9f807b73a79e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -446,7 +446,11 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| a.iter().map(|a| serde_json::from_str(a).unwrap()).collect()); + .map(|a: Vec| { + let result: Result, _> = a.iter().map(|a| serde_json::from_str(a)).collect(); + result + }) + .transpose()?; Ok(OptionalENConfig { filters_limit: load_optional_config_or_default!( @@ -949,7 +953,7 @@ impl RequiredENConfig { eth_client_url: secrets .l1 .as_ref() - .context("Eth config is required")? + .context("L1 secrets are required")? .l1_rpc_url .clone(), main_node_url: en_config.main_node_url.clone(), diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 163835044cac..803d540621dc 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -293,7 +293,13 @@ impl MainNodeBuilder { let circuit_breaker_config = try_load_config!(self.configs.circuit_breaker_config); let with_debug_namespace = state_keeper_config.save_call_traces; - let mut namespaces = Namespace::DEFAULT.to_vec(); + let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { + let result: Result, _> = + namespaces.iter().map(|a| serde_json::from_str(a)).collect(); + result? + } else { + Namespace::DEFAULT.to_vec() + }; if with_debug_namespace { namespaces.push(Namespace::Debug) } diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index 5a3ff7af5bd6..6757ad10526d 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,4 +1,4 @@ l1_chain_id: 9 l2_chain_id: 270 main_node_url: http://localhost:3050 -l1_batch_commit_data_generator_mode: Rollup \ No newline at end of file +l1_batch_commit_data_generator_mode: Rollup From 551dcb7cdaf185a72fb6b01233eef64d81c147c6 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 15:29:12 +0200 Subject: [PATCH 11/35] Support extended_rpc_tracing Signed-off-by: Danil --- core/bin/zksync_server/src/node_builder.rs | 1 + core/node/node_framework/examples/main_node.rs | 1 + .../src/implementations/layers/web3_api/server.rs | 4 ++++ 3 files changed, 6 insertions(+) diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 803d540621dc..e50b8ba03600 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -315,6 +315,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + extended_rpc_tracing: Some(rpc_config.extended_api_tracing), }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, diff --git a/core/node/node_framework/examples/main_node.rs b/core/node/node_framework/examples/main_node.rs index f42cf76d33a2..813e655267d1 100644 --- a/core/node/node_framework/examples/main_node.rs +++ b/core/node/node_framework/examples/main_node.rs @@ -286,6 +286,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + extended_rpc_tracing: Some(rpc_config.extended_api_tracing), }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server.rs b/core/node/node_framework/src/implementations/layers/web3_api/server.rs index 08eaa4b80444..18022b088658 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/server.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/server.rs @@ -29,6 +29,7 @@ pub struct Web3ServerOptionalConfig { pub websocket_requests_per_minute_limit: Option, // used by circuit breaker. pub replication_lag_limit: Option, + pub extended_rpc_tracing: Option, } impl Web3ServerOptionalConfig { @@ -53,6 +54,9 @@ impl Web3ServerOptionalConfig { api_builder = api_builder .with_websocket_requests_per_minute_limit(websocket_requests_per_minute_limit); } + if let Some(extended_rpc_tracing) = self.extended_rpc_tracing { + api_builder = api_builder.with_extended_tracing(extended_rpc_tracing) + } api_builder } } From 69e99d8dfc5241f4e46476842d2d8d0afb219a83 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 16:32:55 +0200 Subject: [PATCH 12/35] Update configs Signed-off-by: Danil --- etc/env/file_based/external_node.yaml | 14 +++++++++++++- etc/env/file_based/general.yaml | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index 6757ad10526d..edb2b4af597f 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,4 +1,16 @@ l1_chain_id: 9 l2_chain_id: 270 -main_node_url: http://localhost:3050 l1_batch_commit_data_generator_mode: Rollup +commitment_generator_max_parallelism: 10 + +main_node_url: http://localhost:3050 +main_node_rate_limit_rps: 1000 +snapshot_recovery: + enabled: true + postgres_max_concurrency: 10 + tree_chunk_size: 200000 +pruning: + enabled: true + chunk_size: 10 + removal_delay_sec: 60 + data_retention_sec: 3600 diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 2b7cfff81a2f..7ed53bf8fed1 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -128,7 +128,7 @@ eth: aggregated_block_execute_deadline: 10 timestamp_criteria_max_allowed_lag: 30 max_eth_tx_data_size: 120000 - aggregated_proof_sizes: 1 + aggregated_proof_sizes: [ 1 ] max_aggregated_tx_gas: 4000000 max_acceptable_priority_fee_in_gwei: 100000000000 pubdata_sending_mode: BLOBS From 116e06b3868474329d61ce03945d384c2babda72 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 16:32:55 +0200 Subject: [PATCH 13/35] Update configs Signed-off-by: Danil --- core/bin/external_node/src/config/observability.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index a4aa9970ca0d..efec5a8a0d87 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -27,7 +27,7 @@ pub(crate) struct ObservabilityENConfig { /// Log format to use: either `plain` (default) or `json`. #[serde(default)] pub log_format: LogFormat, - /// Log directives, same as RUST_LOG + // Log directives in format that is used in `RUST_LOG` pub log_directives: Option, } From f8bdb818857031f78dd36fb996b8539b2e0d03e1 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 3 Jun 2024 17:45:49 +0200 Subject: [PATCH 14/35] Fix tests Signed-off-by: Danil --- core/lib/config/src/testonly.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index b9ed5f272afc..3c6d8d8031ea 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -97,7 +97,8 @@ impl Distribution for EncodeDist { mempool_cache_update_interval: self.sample(rng), mempool_cache_size: self.sample(rng), whitelisted_tokens_for_aa: self.sample_range(rng).map(|_| rng.gen()).collect(), - api_namespaces: self.sample_range(rng).map(|_| self.sample(rng)).collect(), + api_namespaces: self + .sample_opt(|| self.sample_range(rng).map(|_| self.sample(rng)).collect()), extended_api_tracing: self.sample(rng), } } From 92036975c94babefe2dccd98eea9eb08a1846e90 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 4 Jun 2024 10:41:20 +0200 Subject: [PATCH 15/35] Update contract submodule --- contracts | 2 +- core/lib/contracts/src/lib.rs | 2 +- zk_toolbox/crates/config/src/consts.rs | 2 +- .../config/src/forge_interface/script_params.rs | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/contracts b/contracts index 32ca4e665da8..8a70bbbc4812 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 32ca4e665da89f5b4f2f705eee40d91024ad5b48 +Subproject commit 8a70bbbc48125f5bde6189b4e3c6a3ee79631678 diff --git a/core/lib/contracts/src/lib.rs b/core/lib/contracts/src/lib.rs index e27728272150..50fc20c5916f 100644 --- a/core/lib/contracts/src/lib.rs +++ b/core/lib/contracts/src/lib.rs @@ -30,7 +30,7 @@ pub enum ContractLanguage { /// Meanwhile, hardhat has one more intermediate folder. That's why, we have to represent each contract /// by two constants, intermediate folder and actual contract name. For Forge we use only second part const HARDHAT_PATH_PREFIX: &str = "contracts/l1-contracts/artifacts/contracts"; -const FORGE_PATH_PREFIX: &str = "contracts/l1-contracts-foundry/out"; +const FORGE_PATH_PREFIX: &str = "contracts/l1-contracts/out"; const BRIDGEHUB_CONTRACT_FILE: (&str, &str) = ("bridgehub", "IBridgehub.sol/IBridgehub.json"); const STATE_TRANSITION_CONTRACT_FILE: (&str, &str) = ( diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 9082a17abb24..90645ff19acf 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -31,7 +31,7 @@ pub(crate) const LOCAL_DB_PATH: &str = "db/"; pub(crate) const ECOSYSTEM_PATH: &str = "etc/ecosystem"; /// Path to l1 contracts foundry folder inside zksync-era -pub(crate) const L1_CONTRACTS_FOUNDRY: &str = "contracts/l1-contracts-foundry"; +pub(crate) const L1_CONTRACTS_FOUNDRY: &str = "contracts/l1-contracts"; pub(crate) const ERA_CHAIN_ID: ChainId = ChainId(270); diff --git a/zk_toolbox/crates/config/src/forge_interface/script_params.rs b/zk_toolbox/crates/config/src/forge_interface/script_params.rs index a01a15be2a01..0e27871b6708 100644 --- a/zk_toolbox/crates/config/src/forge_interface/script_params.rs +++ b/zk_toolbox/crates/config/src/forge_interface/script_params.rs @@ -29,35 +29,35 @@ impl ForgeScriptParams { pub const DEPLOY_ECOSYSTEM_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-l1.toml", output: "script-out/output-deploy-l1.toml", - script_path: "script/DeployL1.s.sol", + script_path: "deploy_scripts/DeployL1.s.sol", }; pub const INITIALIZE_BRIDGES_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-initialize-shared-bridges.toml", output: "script-out/output-initialize-shared-bridges.toml", - script_path: "script/InitializeSharedBridgeOnL2.sol", + script_path: "deploy_scripts/InitializeSharedBridgeOnL2.sol", }; pub const REGISTER_CHAIN_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/register-hyperchain.toml", output: "script-out/output-register-hyperchain.toml", - script_path: "script/RegisterHyperchain.s.sol", + script_path: "deploy_scripts/RegisterHyperchain.s.sol", }; pub const DEPLOY_ERC20_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-erc20.toml", output: "script-out/output-deploy-erc20.toml", - script_path: "script/DeployErc20.s.sol", + script_path: "deploy_scripts/DeployErc20.s.sol", }; pub const DEPLOY_PAYMASTER_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-paymaster.toml", output: "script-out/output-deploy-paymaster.toml", - script_path: "script/DeployPaymaster.s.sol", + script_path: "deploy_scripts/DeployPaymaster.s.sol", }; pub const ACCEPT_GOVERNANCE_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-accept-admin.toml", output: "script-out/output-accept-admin.toml", - script_path: "script/AcceptAdmin.s.sol", + script_path: "deploy_scripts/AcceptAdmin.s.sol", }; From b2230c1b0a61d60d70a9b6b94f0482318995a707 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 4 Jun 2024 10:42:21 +0200 Subject: [PATCH 16/35] Fix script path --- .../config/src/forge_interface/script_params.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/zk_toolbox/crates/config/src/forge_interface/script_params.rs b/zk_toolbox/crates/config/src/forge_interface/script_params.rs index 0e27871b6708..70ed08ec5651 100644 --- a/zk_toolbox/crates/config/src/forge_interface/script_params.rs +++ b/zk_toolbox/crates/config/src/forge_interface/script_params.rs @@ -29,35 +29,35 @@ impl ForgeScriptParams { pub const DEPLOY_ECOSYSTEM_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-l1.toml", output: "script-out/output-deploy-l1.toml", - script_path: "deploy_scripts/DeployL1.s.sol", + script_path: "deploy-scripts/DeployL1.s.sol", }; pub const INITIALIZE_BRIDGES_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-initialize-shared-bridges.toml", output: "script-out/output-initialize-shared-bridges.toml", - script_path: "deploy_scripts/InitializeSharedBridgeOnL2.sol", + script_path: "deploy-scripts/InitializeSharedBridgeOnL2.sol", }; pub const REGISTER_CHAIN_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/register-hyperchain.toml", output: "script-out/output-register-hyperchain.toml", - script_path: "deploy_scripts/RegisterHyperchain.s.sol", + script_path: "deploy-scripts/RegisterHyperchain.s.sol", }; pub const DEPLOY_ERC20_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-erc20.toml", output: "script-out/output-deploy-erc20.toml", - script_path: "deploy_scripts/DeployErc20.s.sol", + script_path: "deploy-scripts/DeployErc20.s.sol", }; pub const DEPLOY_PAYMASTER_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-deploy-paymaster.toml", output: "script-out/output-deploy-paymaster.toml", - script_path: "deploy_scripts/DeployPaymaster.s.sol", + script_path: "deploy-scripts/DeployPaymaster.s.sol", }; pub const ACCEPT_GOVERNANCE_SCRIPT_PARAMS: ForgeScriptParams = ForgeScriptParams { input: "script-config/config-accept-admin.toml", output: "script-out/output-accept-admin.toml", - script_path: "deploy_scripts/AcceptAdmin.s.sol", + script_path: "deploy-scripts/AcceptAdmin.s.sol", }; From 5837cf0b202b64465690b228065b04f946ee6238 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 13:54:08 +0200 Subject: [PATCH 17/35] Set requires between configs Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 12 ++++++------ core/bin/external_node/src/main.rs | 18 +++++------------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index a588813522ae..62f0b34e5a58 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1203,19 +1203,19 @@ impl ExternalNodeConfig<()> { pub fn from_files( general_config_path: PathBuf, - external_node_config_patch: PathBuf, - secrets_configs: PathBuf, - consensus: Option, + external_node_config_path: PathBuf, + secrets_configs_path: PathBuf, + consensus_config_path: Option, ) -> anyhow::Result { let general_config = read_yaml_repr::(general_config_path) .context("failed decoding general YAML config")?; let external_node_config = - read_yaml_repr::(external_node_config_patch) + read_yaml_repr::(external_node_config_path) .context("failed decoding external node YAML config")?; - let secrets_config = read_yaml_repr::(secrets_configs) + let secrets_config = read_yaml_repr::(secrets_configs_path) .context("failed decoding secrets YAML config")?; - let consensus = consensus + let consensus = consensus_config_path .map(read_yaml_repr::) .transpose() .context("failed decoding consensus YAML config")?; diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 2ace1aad8681..cd400f9cdcc5 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -701,16 +701,15 @@ struct Cli { #[arg(long, default_value = "all")] components: ComponentsToRun, /// Path to the yaml config. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "secrets_path")] config_path: Option, /// Path to the yaml with secrets. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "external_node_config_path")] secrets_path: Option, /// Path to the yaml with genesis. If set, it will be used instead of env vars. - #[arg(long)] + #[arg(long, requires = "config_path")] external_node_config_path: Option, /// Path to the yaml with consensus. - #[arg(long)] consensus_path: Option, } @@ -769,15 +768,8 @@ async fn main() -> anyhow::Result<()> { let opt = Cli::parse(); let mut config = if let Some(config_path) = opt.config_path.clone() { - let secrets_path = opt - .secrets_path - .clone() - .context("File based config can't be used partially please provide all paths")?; - let external_node_config_path = opt - .external_node_config_path - .clone() - .context("File based config can't be used partially please provide all paths")?; - + let secrets_path = opt.secrets_path.clone().unwrap(); + let external_node_config_path = opt.external_node_config_path.clone().unwrap(); ExternalNodeConfig::from_files( config_path, external_node_config_path, From 9ee546b064792e8eaca6502856ede8d53acfccee Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 15:56:26 +0200 Subject: [PATCH 18/35] Add commitment generator component Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 16 +++++++++---- .../src/configs/commitment_generator.rs | 8 +++++++ core/lib/config/src/configs/en_config.rs | 5 +--- core/lib/config/src/configs/experimental.rs | 2 +- core/lib/config/src/configs/general.rs | 7 +++--- core/lib/config/src/configs/mod.rs | 2 ++ .../src/commitment_generator.rs | 24 +++++++++++++++++++ core/lib/protobuf_config/src/en.rs | 10 +------- core/lib/protobuf_config/src/general.rs | 5 +++- core/lib/protobuf_config/src/lib.rs | 1 + .../proto/config/commitment_generator.proto | 7 ++++++ .../protobuf_config/src/proto/config/en.proto | 2 -- .../src/proto/config/general.proto | 2 ++ .../src/temp_config_store/mod.rs | 1 + 14 files changed, 67 insertions(+), 25 deletions(-) create mode 100644 core/lib/config/src/configs/commitment_generator.rs create mode 100644 core/lib/protobuf_config/src/commitment_generator.rs create mode 100644 core/lib/protobuf_config/src/proto/config/commitment_generator.proto diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 62f0b34e5a58..76df29afc95a 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1089,8 +1089,10 @@ impl ExperimentalENConfig { tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), - commitment_generator_max_parallelism: external_node_config - .commitment_generator_max_parallelism, + commitment_generator_max_parallelism: general_config + .commitment_generator + .as_ref() + .map(|a| a.max_parallelism), }) } } @@ -1142,9 +1144,13 @@ pub struct ApiComponentConfig { } impl ApiComponentConfig { - fn from_configs(en_config: &ENConfig) -> Self { + fn from_configs(general_config: &GeneralConfig) -> Self { ApiComponentConfig { - tree_api_remote_url: en_config.tree_api_remote_url.clone(), + tree_api_remote_url: general_config + .api_config + .as_ref() + .map(|a| a.web3_json_rpc.tree_api_url.clone()) + .flatten(), } } } @@ -1244,7 +1250,7 @@ impl ExternalNodeConfig<()> { let experimental = ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; - let api_component = ApiComponentConfig::from_configs(&external_node_config); + let api_component = ApiComponentConfig::from_configs(&general_config); let tree_component = TreeComponentConfig::from_configs(&general_config); Ok(Self { diff --git a/core/lib/config/src/configs/commitment_generator.rs b/core/lib/config/src/configs/commitment_generator.rs new file mode 100644 index 000000000000..4a06bfdba63c --- /dev/null +++ b/core/lib/config/src/configs/commitment_generator.rs @@ -0,0 +1,8 @@ +use std::num::NonZeroU32; + +use serde::Deserialize; + +#[derive(Debug, Deserialize, Clone, PartialEq)] +pub struct CommitmentGeneratorConfig { + pub max_parallelism: NonZeroU32, +} diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index ab8e49617e84..5f003650fd41 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,4 +1,4 @@ -use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize}; +use std::num::{NonZeroU64, NonZeroUsize}; use serde::Deserialize; use zksync_basic_types::{ @@ -32,9 +32,6 @@ pub struct ENConfig { pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, - // En specific components - pub commitment_generator_max_parallelism: Option, - pub tree_api_remote_url: Option, pub snapshot_recovery: Option, pub pruning: Option, } diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 31bdf3a4730f..19e08dc2bb8a 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -36,7 +36,7 @@ impl Default for ExperimentalDBConfig { Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, reads_persistence_enabled: Self::default_protective_reads_persistence_enabled(), - processing_delay_ms: 0, + processing_delay_ms: Self::default_merkle_tree_processing_delay_ms(), include_indices_and_filters_in_block_cache: false, } } diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index ef02f557bc18..d5c8490ab2be 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -4,9 +4,9 @@ use crate::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, vm_runner::ProtectiveReadsWriterConfig, - FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig, - PrometheusConfig, ProofDataHandlerConfig, + CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, PostgresConfig, SnapshotsCreatorConfig, }; @@ -34,4 +34,5 @@ pub struct GeneralConfig { pub snapshot_creator: Option, pub observability: Option, pub protective_reads_writer_config: Option, + pub commitment_generator: Option, } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 990b0c858b2c..ae988abca2ce 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -1,6 +1,7 @@ // Public re-exports pub use self::{ api::ApiConfig, + commitment_generator::CommitmentGeneratorConfig, contract_verifier::ContractVerifierConfig, contracts::{ContractsConfig, EcosystemContracts}, database::{DBConfig, PostgresConfig}, @@ -25,6 +26,7 @@ pub use self::{ pub mod api; pub mod chain; +mod commitment_generator; pub mod consensus; pub mod contract_verifier; pub mod contracts; diff --git a/core/lib/protobuf_config/src/commitment_generator.rs b/core/lib/protobuf_config/src/commitment_generator.rs new file mode 100644 index 000000000000..23af3ccce76e --- /dev/null +++ b/core/lib/protobuf_config/src/commitment_generator.rs @@ -0,0 +1,24 @@ +use std::num::NonZeroU32; + +use anyhow::Context as _; +use zksync_config::configs::CommitmentGeneratorConfig; +use zksync_protobuf::{repr::ProtoRepr, required}; + +use crate::proto::commitment_generator as proto; + +impl ProtoRepr for proto::CommitmentGenerator { + type Type = CommitmentGeneratorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + max_parallelism: NonZeroU32::new( + *required(&self.max_parallelism).context("max_parallelism")?, + ) + .context("cannot be 0")?, + }) + } + fn build(this: &Self::Type) -> Self { + Self { + max_parallelism: Some(this.max_parallelism.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 8e3b99d1aaa8..4dbcae4fdbdd 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,5 +1,5 @@ use std::{ - num::{NonZeroU32, NonZeroU64, NonZeroUsize}, + num::{NonZeroU64, NonZeroUsize}, str::FromStr, }; @@ -74,10 +74,6 @@ impl ProtoRepr for proto::ExternalNode { .and_then(|x| Ok(crate::proto::genesis::L1BatchCommitDataGeneratorMode::try_from(*x)?)) .context("l1_batch_commit_data_generator_mode")? .parse(), - commitment_generator_max_parallelism: self - .commitment_generator_max_parallelism - .and_then(NonZeroU32::new), - tree_api_remote_url: self.tree_api_remote_url.clone(), main_node_rate_limit_rps: self .main_node_rate_limit_rps .and_then(|a| NonZeroUsize::new(a as usize)), @@ -98,10 +94,6 @@ impl ProtoRepr for proto::ExternalNode { ) .into(), ), - tree_api_remote_url: this.tree_api_remote_url.clone(), - commitment_generator_max_parallelism: this - .commitment_generator_max_parallelism - .map(|a| a.get()), main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), pruning: this.pruning.as_ref().map(ProtoRepr::build), diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index ba2076a09a14..3583d77e5ce3 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -38,7 +38,9 @@ impl ProtoRepr for proto::GeneralConfig { .context("snapshot_creator")?, observability: read_optional_repr(&self.observability).context("observability")?, protective_reads_writer_config: read_optional_repr(&self.protective_reads_writer) - .context("vm_runner")?, + .context("protective_reads_writer_config")?, + commitment_generator: read_optional_repr(&self.commitment_generator) + .context("commitment_generator")?, }) } @@ -74,6 +76,7 @@ impl ProtoRepr for proto::GeneralConfig { .protective_reads_writer_config .as_ref() .map(ProtoRepr::build), + commitment_generator: this.commitment_generator.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index ebc0ae6b85a2..8c95f4cedf98 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -7,6 +7,7 @@ mod api; mod chain; mod circuit_breaker; +mod commitment_generator; mod consensus; mod contract_verifier; mod contracts; diff --git a/core/lib/protobuf_config/src/proto/config/commitment_generator.proto b/core/lib/protobuf_config/src/proto/config/commitment_generator.proto new file mode 100644 index 000000000000..62b9566e1866 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/commitment_generator.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package zksync.config.commitment_generator; + +message CommitmentGenerator { + optional uint32 max_parallelism = 1; +} diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 375872d36df4..7b64f75f270d 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -20,8 +20,6 @@ message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required - optional uint32 commitment_generator_max_parallelism = 4; // optional - optional string tree_api_remote_url = 5; // optional optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup optional SnapshotRecovery snapshot_recovery = 8; diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index b606417d129a..7dcedefd6a4d 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -14,6 +14,7 @@ import "zksync/config/observability.proto"; import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; import "zksync/config/vm_runner.proto"; +import "zksync/config/commitment_generator.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -37,4 +38,5 @@ message GeneralConfig { optional config.snapshot_creator.SnapshotsCreator snapshot_creator = 31; optional config.observability.Observability observability = 32; optional config.vm_runner.ProtectiveReadsWriter protective_reads_writer = 33; + optional config.commitment_generator.CommitmentGenerator commitment_generator = 34; } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index c1c8af29d964..fc9b165d782f 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -97,6 +97,7 @@ impl TempConfigStore { snapshot_creator: self.snapshot_creator.clone(), observability: self.observability.clone(), protective_reads_writer_config: self.protective_reads_writer_config.clone(), + commitment_generator: None, } } From e2fe4814e991bfdbfa473c939f82070d2729a2e8 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 17:46:21 +0200 Subject: [PATCH 19/35] Move snapshot recovery and prunning to the general config Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 25 ++++---- core/bin/zksync_server/src/main.rs | 3 + core/lib/config/src/configs/en_config.rs | 20 +------ core/lib/config/src/configs/general.rs | 4 ++ core/lib/config/src/configs/mod.rs | 4 ++ core/lib/config/src/configs/pruning.rs | 11 ++++ .../config/src/configs/snapshot_recovery.rs | 10 ++++ core/lib/protobuf_config/src/en.rs | 58 +------------------ core/lib/protobuf_config/src/general.rs | 5 ++ core/lib/protobuf_config/src/lib.rs | 3 + .../protobuf_config/src/proto/config/en.proto | 15 ----- .../src/proto/config/general.proto | 4 ++ .../src/proto/config/pruning.proto | 10 ++++ .../src/proto/config/snapshot_recovery.proto | 10 ++++ core/lib/protobuf_config/src/pruning.rs | 28 +++++++++ .../protobuf_config/src/snapshot_recovery.rs | 28 +++++++++ .../src/temp_config_store/mod.rs | 14 +++-- 17 files changed, 144 insertions(+), 108 deletions(-) create mode 100644 core/lib/config/src/configs/pruning.rs create mode 100644 core/lib/config/src/configs/snapshot_recovery.rs create mode 100644 core/lib/protobuf_config/src/proto/config/pruning.proto create mode 100644 core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto create mode 100644 core/lib/protobuf_config/src/pruning.rs create mode 100644 core/lib/protobuf_config/src/snapshot_recovery.rs diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 76df29afc95a..4c46265a79a5 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -607,33 +607,33 @@ impl OptionalENConfig { default_l2_block_seal_queue_capacity ), l1_batch_commit_data_generator_mode: enconfig.l1_batch_commit_data_generator_mode, - snapshots_recovery_enabled: enconfig + snapshots_recovery_enabled: general_config .snapshot_recovery .as_ref() .map(|a| a.enabled) .unwrap_or_default(), snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( - enconfig.snapshot_recovery, + general_config.snapshot_recovery, postgres_max_concurrency, default_snapshots_recovery_postgres_max_concurrency ), - pruning_enabled: enconfig + pruning_enabled: general_config .pruning .as_ref() .map(|a| a.enabled) .unwrap_or_default(), pruning_chunk_size: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, chunk_size, default_pruning_chunk_size ), pruning_removal_delay_sec: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, removal_delay_sec, default_pruning_removal_delay_sec ), pruning_data_retention_sec: load_optional_config_or_default!( - enconfig.pruning, + general_config.pruning, data_retention_sec, default_pruning_data_retention_sec ), @@ -1069,10 +1069,7 @@ impl ExperimentalENConfig { self.state_keeper_db_block_cache_capacity_mb * BYTES_IN_MEGABYTE } - pub fn from_configs( - general_config: &GeneralConfig, - external_node_config: &ENConfig, - ) -> anyhow::Result { + pub fn from_configs(general_config: &GeneralConfig) -> anyhow::Result { Ok(Self { state_keeper_db_block_cache_capacity_mb: load_config_or_default!( general_config.db_config, @@ -1085,7 +1082,7 @@ impl ExperimentalENConfig { experimental.state_keeper_db_max_open_files ), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( - external_node_config.snapshot_recovery, + general_config.snapshot_recovery, tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), @@ -1149,8 +1146,7 @@ impl ApiComponentConfig { tree_api_remote_url: general_config .api_config .as_ref() - .map(|a| a.web3_json_rpc.tree_api_url.clone()) - .flatten(), + .and_then(|a| a.web3_json_rpc.tree_api_url.clone()), } } } @@ -1247,8 +1243,7 @@ impl ExternalNodeConfig<()> { .max_connections()?, }; let observability = ObservabilityENConfig::from_configs(&general_config)?; - let experimental = - ExperimentalENConfig::from_configs(&general_config, &external_node_config)?; + let experimental = ExperimentalENConfig::from_configs(&general_config)?; let api_component = ApiComponentConfig::from_configs(&general_config); let tree_component = TreeComponentConfig::from_configs(&general_config); diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index f1eedd592386..0c6e05c2059a 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -308,5 +308,8 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + commitment_generator: None, + pruning: None, + snapshot_recovery: None, }) } diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 5f003650fd41..32dc5b7c7b49 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,25 +1,10 @@ -use std::num::{NonZeroU64, NonZeroUsize}; +use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::{ commitment::L1BatchCommitmentMode, url::SensitiveUrl, L1ChainId, L2ChainId, }; -#[derive(Debug, Clone, PartialEq, Deserialize)] -pub struct Pruning { - pub enabled: bool, - pub chunk_size: Option, - pub removal_delay_sec: Option, - pub data_retention_sec: Option, -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -pub struct SnapshotRecovery { - pub enabled: bool, - pub postgres_max_concurrency: Option, - pub tree_chunk_size: Option, -} - /// Temporary config for initializing external node, will be completely replaced by consensus config later #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct ENConfig { @@ -31,7 +16,4 @@ pub struct ENConfig { // Main node configuration pub main_node_url: SensitiveUrl, pub main_node_rate_limit_rps: Option, - - pub snapshot_recovery: Option, - pub pruning: Option, } diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index d5c8490ab2be..879ff31d5046 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -3,6 +3,8 @@ use crate::{ chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig}, fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, + pruning::PruningConfig, + snapshot_recovery::SnapshotRecoveryConfig, vm_runner::ProtectiveReadsWriterConfig, CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, @@ -35,4 +37,6 @@ pub struct GeneralConfig { pub observability: Option, pub protective_reads_writer_config: Option, pub commitment_generator: Option, + pub snapshot_recovery: Option, + pub pruning: Option, } diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index ae988abca2ce..9e04f483357f 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -18,7 +18,9 @@ pub use self::{ object_store::ObjectStoreConfig, observability::{ObservabilityConfig, OpentelemetryConfig}, proof_data_handler::ProofDataHandlerConfig, + pruning::PruningConfig, secrets::{DatabaseSecrets, L1Secrets, Secrets}, + snapshot_recovery::SnapshotRecoveryConfig, snapshots_creator::SnapshotsCreatorConfig, utils::PrometheusConfig, vm_runner::ProtectiveReadsWriterConfig, @@ -47,7 +49,9 @@ pub mod house_keeper; pub mod object_store; pub mod observability; pub mod proof_data_handler; +pub mod pruning; pub mod secrets; +pub mod snapshot_recovery; pub mod snapshots_creator; pub mod utils; pub mod vm_runner; diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs new file mode 100644 index 000000000000..ae09d43b3bfc --- /dev/null +++ b/core/lib/config/src/configs/pruning.rs @@ -0,0 +1,11 @@ +use std::num::NonZeroU64; + +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct PruningConfig { + pub enabled: bool, + pub chunk_size: Option, + pub removal_delay_sec: Option, + pub data_retention_sec: Option, +} diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs new file mode 100644 index 000000000000..96389eb2fd2e --- /dev/null +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -0,0 +1,10 @@ +use std::num::NonZeroUsize; + +use serde::Deserialize; + +#[derive(Debug, Clone, PartialEq, Deserialize)] +pub struct SnapshotRecoveryConfig { + pub enabled: bool, + pub postgres_max_concurrency: Option, + pub tree_chunk_size: Option, +} diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 4dbcae4fdbdd..b72a5b142cfb 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,58 +1,11 @@ -use std::{ - num::{NonZeroU64, NonZeroUsize}, - str::FromStr, -}; +use std::{num::NonZeroUsize, str::FromStr}; use anyhow::Context; use zksync_basic_types::{url::SensitiveUrl, L1ChainId, L2ChainId}; -use zksync_config::configs::en_config::{ENConfig, Pruning, SnapshotRecovery}; +use zksync_config::configs::en_config::ENConfig; use zksync_protobuf::{required, ProtoRepr}; -use crate::{proto::en as proto, read_optional_repr}; - -impl ProtoRepr for proto::Pruning { - type Type = Pruning; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - chunk_size: self.chunk_size, - removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), - data_retention_sec: self.data_retention_sec, - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - enabled: Some(this.enabled), - chunk_size: this.chunk_size, - removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), - data_retention_sec: this.data_retention_sec, - } - } -} - -impl ProtoRepr for proto::SnapshotRecovery { - type Type = SnapshotRecovery; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - postgres_max_concurrency: self - .postgres_max_concurrency - .and_then(|a| NonZeroUsize::new(a as usize)), - tree_chunk_size: self.tree_chunk_size, - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - enabled: Some(this.enabled), - postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), - tree_chunk_size: this.tree_chunk_size, - } - } -} +use crate::proto::en as proto; impl ProtoRepr for proto::ExternalNode { type Type = ENConfig; @@ -77,9 +30,6 @@ impl ProtoRepr for proto::ExternalNode { main_node_rate_limit_rps: self .main_node_rate_limit_rps .and_then(|a| NonZeroUsize::new(a as usize)), - pruning: read_optional_repr(&self.pruning).context("pruning")?, - snapshot_recovery: read_optional_repr(&self.snapshot_recovery) - .context("snapshot_recovery")?, }) } @@ -95,8 +45,6 @@ impl ProtoRepr for proto::ExternalNode { .into(), ), main_node_rate_limit_rps: this.main_node_rate_limit_rps.map(|a| a.get() as u32), - snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), - pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index 3583d77e5ce3..1cb62b83f155 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -41,6 +41,9 @@ impl ProtoRepr for proto::GeneralConfig { .context("protective_reads_writer_config")?, commitment_generator: read_optional_repr(&self.commitment_generator) .context("commitment_generator")?, + pruning: read_optional_repr(&self.pruning).context("pruning")?, + snapshot_recovery: read_optional_repr(&self.snapshot_recovery) + .context("snapshot_recovery")?, }) } @@ -77,6 +80,8 @@ impl ProtoRepr for proto::GeneralConfig { .as_ref() .map(ProtoRepr::build), commitment_generator: this.commitment_generator.as_ref().map(ProtoRepr::build), + snapshot_recovery: this.snapshot_recovery.as_ref().map(ProtoRepr::build), + pruning: this.pruning.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs index 8c95f4cedf98..14e4f5455f5f 100644 --- a/core/lib/protobuf_config/src/lib.rs +++ b/core/lib/protobuf_config/src/lib.rs @@ -23,8 +23,11 @@ mod observability; mod proof_data_handler; pub mod proto; mod prover; +mod pruning; mod secrets; mod snapshots_creator; + +mod snapshot_recovery; pub mod testonly; #[cfg(test)] mod tests; diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index 7b64f75f270d..ac7cb59b156a 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -3,25 +3,10 @@ import "zksync/config/genesis.proto"; package zksync.config.en; -message SnapshotRecovery { - optional bool enabled = 1; - optional uint64 postgres_max_concurrency = 2; - optional uint64 tree_chunk_size = 3; -} - -message Pruning { - optional bool enabled = 1; - optional uint32 chunk_size = 2; - optional uint64 removal_delay_sec = 3; - optional uint64 data_retention_sec = 4; -} - message ExternalNode { optional string main_node_url = 1; // required optional uint64 l2_chain_id = 2; // required optional uint64 l1_chain_id = 3; // required optional uint32 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup - optional SnapshotRecovery snapshot_recovery = 8; - optional Pruning pruning = 9; } diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index 7dcedefd6a4d..5465a441ad54 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -15,6 +15,8 @@ import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; import "zksync/config/vm_runner.proto"; import "zksync/config/commitment_generator.proto"; +import "zksync/config/snapshot_recovery.proto"; +import "zksync/config/pruning.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -39,4 +41,6 @@ message GeneralConfig { optional config.observability.Observability observability = 32; optional config.vm_runner.ProtectiveReadsWriter protective_reads_writer = 33; optional config.commitment_generator.CommitmentGenerator commitment_generator = 34; + optional config.snapshot_recovery.SnapshotRecovery snapshot_recovery = 35; + optional config.pruning.Pruning pruning = 36; } diff --git a/core/lib/protobuf_config/src/proto/config/pruning.proto b/core/lib/protobuf_config/src/proto/config/pruning.proto new file mode 100644 index 000000000000..351f353bf060 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/pruning.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.pruning; + +message Pruning { + optional bool enabled = 1; + optional uint32 chunk_size = 2; + optional uint64 removal_delay_sec = 3; + optional uint64 data_retention_sec = 4; +} diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto new file mode 100644 index 000000000000..da4b4b585bff --- /dev/null +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +package zksync.config.snapshot_recovery; + +message SnapshotRecovery { + optional bool enabled = 1; + optional uint64 postgres_max_concurrency = 2; + optional uint64 tree_chunk_size = 3; +} + diff --git a/core/lib/protobuf_config/src/pruning.rs b/core/lib/protobuf_config/src/pruning.rs new file mode 100644 index 000000000000..ed0ebb10b92f --- /dev/null +++ b/core/lib/protobuf_config/src/pruning.rs @@ -0,0 +1,28 @@ +use std::num::NonZeroU64; + +use zksync_config::configs::PruningConfig; +use zksync_protobuf::ProtoRepr; + +use crate::proto::pruning as proto; + +impl ProtoRepr for proto::Pruning { + type Type = PruningConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + chunk_size: self.chunk_size, + removal_delay_sec: self.removal_delay_sec.and_then(NonZeroU64::new), + data_retention_sec: self.data_retention_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + chunk_size: this.chunk_size, + removal_delay_sec: this.removal_delay_sec.map(|a| a.get()), + data_retention_sec: this.data_retention_sec, + } + } +} diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs new file mode 100644 index 000000000000..19e2862d4e99 --- /dev/null +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -0,0 +1,28 @@ +use std::num::NonZeroUsize; + +use zksync_config::configs::SnapshotRecoveryConfig; +use zksync_protobuf::ProtoRepr; + +use crate::proto::snapshot_recovery as proto; + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecoveryConfig; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + postgres_max_concurrency: self + .postgres_max_concurrency + .and_then(|a| NonZeroUsize::new(a as usize)), + tree_chunk_size: self.tree_chunk_size, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + enabled: Some(this.enabled), + postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), + tree_chunk_size: this.tree_chunk_size, + } + } +} diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index fc9b165d782f..6476172c3767 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -11,9 +11,10 @@ use zksync_config::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, wallets::{AddressWallet, EthSender, StateKeeper, Wallet, Wallets}, - FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, - FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, GeneralConfig, - ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, + CommitmentGeneratorConfig, FriProofCompressorConfig, FriProverConfig, + FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, + GeneralConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, + ProtectiveReadsWriterConfig, PruningConfig, SnapshotRecoveryConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -71,6 +72,9 @@ pub struct TempConfigStore { pub observability: Option, pub snapshot_creator: Option, pub protective_reads_writer_config: Option, + pub commitment_generator: Option, + pub pruning: Option, + pub snapshot_recovery: Option, } impl TempConfigStore { @@ -97,7 +101,9 @@ impl TempConfigStore { snapshot_creator: self.snapshot_creator.clone(), observability: self.observability.clone(), protective_reads_writer_config: self.protective_reads_writer_config.clone(), - commitment_generator: None, + commitment_generator: self.commitment_generator.clone(), + snapshot_recovery: self.snapshot_recovery.clone(), + pruning: self.pruning.clone(), } } From 71dff3237272d60eb6f3ddbb17dc38b083c2279e Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 4 Jun 2024 18:13:21 +0200 Subject: [PATCH 20/35] Fix file based config Signed-off-by: Danil --- etc/env/file_based/external_node.yaml | 10 ---------- etc/env/file_based/general.yaml | 13 +++++++++++++ 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/etc/env/file_based/external_node.yaml b/etc/env/file_based/external_node.yaml index edb2b4af597f..675baf739686 100644 --- a/etc/env/file_based/external_node.yaml +++ b/etc/env/file_based/external_node.yaml @@ -1,16 +1,6 @@ l1_chain_id: 9 l2_chain_id: 270 l1_batch_commit_data_generator_mode: Rollup -commitment_generator_max_parallelism: 10 main_node_url: http://localhost:3050 main_node_rate_limit_rps: 1000 -snapshot_recovery: - enabled: true - postgres_max_concurrency: 10 - tree_chunk_size: 200000 -pruning: - enabled: true - chunk_size: 10 - removal_delay_sec: 60 - data_retention_sec: 3600 diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index f1ed009f01c7..5766af85c683 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -325,3 +325,16 @@ observability: protective_reads_writer: protective_reads_db_path: "./db/main/protective_reads" protective_reads_window_size: 3 + +snapshot_recovery: + enabled: true + postgres_max_concurrency: 10 + tree_chunk_size: 200000 +pruning: + enabled: true + chunk_size: 10 + removal_delay_sec: 60 + data_retention_sec: 3600 + +commitment_generator: + max_parallelism: 10 From b33173c4f8b3973fc5c3b5a88b9d5fc28ee5cf76 Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 6 Jun 2024 15:09:38 +0400 Subject: [PATCH 21/35] Support for api namespaces Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 5 +---- core/bin/zksync_server/src/node_builder.rs | 4 +--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 4c46265a79a5..a574a01ad291 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -444,10 +444,7 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| { - let result: Result, _> = a.iter().map(|a| serde_json::from_str(a)).collect(); - result - }) + .map(|a: Vec| serde_json::from_str(&a.join(","))) .transpose()?; Ok(OptionalENConfig { diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 1dc9e1de554e..548445b02a4e 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -295,9 +295,7 @@ impl MainNodeBuilder { let with_debug_namespace = state_keeper_config.save_call_traces; let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { - let result: Result, _> = - namespaces.iter().map(|a| serde_json::from_str(a)).collect(); - result? + serde_json::from_str(&namespaces.join(","))? } else { Namespace::DEFAULT.to_vec() }; From 0583b69ef4d820edd1ac7e4d96acd7cd07e608ef Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 6 Jun 2024 16:47:32 +0400 Subject: [PATCH 22/35] Fix general.yaml Signed-off-by: Danil --- etc/env/file_based/general.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 61c67586b1a7..3ceda661cff5 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -63,6 +63,7 @@ api: estimate_gas_scale_factor: 1.2 estimate_gas_acceptable_overestimation: 1000 max_tx_size: 1000000 + api_namespaces: [ eth,net,web3,zks,pubsub ] max_response_body_size_overrides: - method: eth_getTransactionReceipt # no size specified, meaning no size limit - method: zks_getProof From 0d39b60693482dac486cc970684be0b435a16db5 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 10:53:27 +0400 Subject: [PATCH 23/35] Update snapshot recovery Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 5 +++++ core/lib/config/src/configs/snapshot_recovery.rs | 3 +++ .../src/proto/config/snapshot_recovery.proto | 2 ++ core/lib/protobuf_config/src/snapshot_recovery.rs | 10 ++++++++++ 4 files changed, 20 insertions(+) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index d1ed4dea1ec9..e61dac88bee8 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -1088,11 +1088,16 @@ impl ExperimentalENConfig { general_config.db_config, experimental.state_keeper_db_max_open_files ), + snapshots_recovery_l1_batch: load_config!(general_config.snapshot_recovery, l1_batch), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( general_config.snapshot_recovery, tree_chunk_size, default_snapshots_recovery_tree_chunk_size ), + snapshots_recovery_tree_parallel_persistence_buffer: load_config!( + general_config.snapshot_recovery, + tree_parallel_persistence_buffer + ), commitment_generator_max_parallelism: general_config .commitment_generator .as_ref() diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index 96389eb2fd2e..1ac5d6a559c9 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -1,10 +1,13 @@ use std::num::NonZeroUsize; use serde::Deserialize; +use zksync_basic_types::L1BatchNumber; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct SnapshotRecoveryConfig { pub enabled: bool, pub postgres_max_concurrency: Option, pub tree_chunk_size: Option, + pub l1_batch: Option, + pub tree_parallel_persistence_buffer: Option, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index da4b4b585bff..f74282608d1a 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -6,5 +6,7 @@ message SnapshotRecovery { optional bool enabled = 1; optional uint64 postgres_max_concurrency = 2; optional uint64 tree_chunk_size = 3; + optional uint32 l1_batch = 4; + optional uint64 tree_parallel_persistence_buffer = 5; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 19e2862d4e99..269261ed54a2 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -1,5 +1,6 @@ use std::num::NonZeroUsize; +use zksync_basic_types::L1BatchNumber; use zksync_config::configs::SnapshotRecoveryConfig; use zksync_protobuf::ProtoRepr; @@ -15,6 +16,11 @@ impl ProtoRepr for proto::SnapshotRecovery { .postgres_max_concurrency .and_then(|a| NonZeroUsize::new(a as usize)), tree_chunk_size: self.tree_chunk_size, + tree_parallel_persistence_buffer: self + .tree_parallel_persistence_buffer + .and_then(|a| NonZeroUsize::new(a as usize)), + + l1_batch: self.l1_batch.map(L1BatchNumber), }) } @@ -23,6 +29,10 @@ impl ProtoRepr for proto::SnapshotRecovery { enabled: Some(this.enabled), postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), tree_chunk_size: this.tree_chunk_size, + l1_batch: this.l1_batch.map(|a| a.0), + tree_parallel_persistence_buffer: this + .tree_parallel_persistence_buffer + .map(|a| a.get() as u64), } } } From af21794836cf8cdfed9b32cf71c4130475eb8347 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 14:26:43 +0400 Subject: [PATCH 24/35] Split snapshot recovery to smaller files Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 6 +- core/lib/config/src/configs/experimental.rs | 2 + core/lib/config/src/configs/pruning.rs | 9 +++ .../config/src/configs/snapshot_recovery.rs | 34 +++++++- .../src/proto/config/snapshot_recovery.proto | 14 +++- .../protobuf_config/src/snapshot_recovery.rs | 77 +++++++++++++++---- .../src/temp_config_store/mod.rs | 2 +- 7 files changed, 119 insertions(+), 25 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e61dac88bee8..10e32c17ca69 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -611,7 +611,7 @@ impl OptionalENConfig { .unwrap_or_default(), snapshots_recovery_postgres_max_concurrency: load_optional_config_or_default!( general_config.snapshot_recovery, - postgres_max_concurrency, + postgres.max_concurrency, default_snapshots_recovery_postgres_max_concurrency ), pruning_enabled: general_config @@ -1091,12 +1091,12 @@ impl ExperimentalENConfig { snapshots_recovery_l1_batch: load_config!(general_config.snapshot_recovery, l1_batch), snapshots_recovery_tree_chunk_size: load_optional_config_or_default!( general_config.snapshot_recovery, - tree_chunk_size, + tree.chunk_size, default_snapshots_recovery_tree_chunk_size ), snapshots_recovery_tree_parallel_persistence_buffer: load_config!( general_config.snapshot_recovery, - tree_parallel_persistence_buffer + tree.parallel_persistence_buffer ), commitment_generator_max_parallelism: general_config .commitment_generator diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 19e08dc2bb8a..46d93cdf18f6 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -50,9 +50,11 @@ impl ExperimentalDBConfig { pub fn state_keeper_db_block_cache_capacity(&self) -> usize { self.state_keeper_db_block_cache_capacity_mb * super::BYTES_IN_MEGABYTE } + const fn default_protective_reads_persistence_enabled() -> bool { true } + const fn default_merkle_tree_processing_delay_ms() -> u64 { 100 } diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs index ae09d43b3bfc..f1b3a8e74558 100644 --- a/core/lib/config/src/configs/pruning.rs +++ b/core/lib/config/src/configs/pruning.rs @@ -5,7 +5,16 @@ use serde::Deserialize; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct PruningConfig { pub enabled: bool, + /// Chunk size for multi-get operations. Can speed up loading data for the Merkle tree on some environments, + /// but the effects vary wildly depending on the setup (e.g., the filesystem used). pub chunk_size: Option, + /// Delta between soft- and hard-removing data from Postgres. Should be reasonably large (order of 60 seconds). + /// The default value is 60 seconds. pub removal_delay_sec: Option, + /// If set, L1 batches will be pruned after the batch timestamp is this old (in seconds). Note that an L1 batch + /// may be temporarily retained for other reasons; e.g., a batch cannot be pruned until it is executed on L1, + /// which happens roughly 24 hours after its generation on the mainnet. Thus, in practice this value can specify + /// the retention period greater than that implicitly imposed by other criteria (e.g., 7 or 30 days). + /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 1 hour. pub data_retention_sec: Option, } diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index 1ac5d6a559c9..d2cf0da190d2 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -3,11 +3,39 @@ use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::L1BatchNumber; +#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +pub struct Tree { + /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. + /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds). + /// + /// **Important.** This value cannot be changed in the middle of tree recovery (i.e., if a node is stopped in the middle + /// of recovery and then restarted with a different config). + pub chunk_size: Option, + /// Buffer capacity for parallel persistence operations. Should be reasonably small since larger buffer means more RAM usage; + /// buffer elements are persisted tree chunks. OTOH, small buffer can lead to persistence parallelization being inefficient. + /// + /// If not set, parallel persistence will be disabled. + pub parallel_persistence_buffer: Option, +} + +#[derive(Debug, Clone, PartialEq, Deserialize, Default)] +pub struct Postgres { + /// Maximum concurrency factor for the concurrent parts of snapshot recovery for Postgres. It may be useful to + /// reduce this factor to about 5 if snapshot recovery overloads I/O capacity of the node. Conversely, + /// if I/O capacity of your infra is high, you may increase concurrency to speed up Postgres recovery. + pub max_concurrency: Option, +} + #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct SnapshotRecoveryConfig { + /// Enables application-level snapshot recovery. Required to start a node that was recovered from a snapshot, + /// or to initialize a node from a snapshot. Has no effect if a node that was initialized from a Postgres dump + /// or was synced from genesis. + /// + /// This is an experimental and incomplete feature; do not use unless you know what you're doing. pub enabled: bool, - pub postgres_max_concurrency: Option, - pub tree_chunk_size: Option, + /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing. pub l1_batch: Option, - pub tree_parallel_persistence_buffer: Option, + pub tree: Tree, + pub postgres: Postgres, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index f74282608d1a..5f217723f8a7 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -2,11 +2,19 @@ syntax = "proto3"; package zksync.config.snapshot_recovery; +message Tree { + optional uint64 chunk_size = 1; + optional uint64 parallel_persistence_buffer = 2; +} + +message Postgres { + optional uint64 max_concurrency = 1; +} + message SnapshotRecovery { optional bool enabled = 1; - optional uint64 postgres_max_concurrency = 2; - optional uint64 tree_chunk_size = 3; + optional Postgres postgres = 2; + optional Tree tree = 3; optional uint32 l1_batch = 4; - optional uint64 tree_parallel_persistence_buffer = 5; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 269261ed54a2..7a84d1a158c6 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -1,38 +1,85 @@ use std::num::NonZeroUsize; +use anyhow::Context; use zksync_basic_types::L1BatchNumber; -use zksync_config::configs::SnapshotRecoveryConfig; +use zksync_config::configs::{ + snapshot_recovery::{Postgres, Tree}, + SnapshotRecoveryConfig, +}; use zksync_protobuf::ProtoRepr; -use crate::proto::snapshot_recovery as proto; +use crate::{proto::snapshot_recovery as proto, read_optional_repr}; -impl ProtoRepr for proto::SnapshotRecovery { - type Type = SnapshotRecoveryConfig; +impl ProtoRepr for proto::Tree { + type Type = Tree; fn read(&self) -> anyhow::Result { Ok(Self::Type { - enabled: self.enabled.unwrap_or_default(), - postgres_max_concurrency: self - .postgres_max_concurrency + chunk_size: self.chunk_size, + parallel_persistence_buffer: self + .parallel_persistence_buffer .and_then(|a| NonZeroUsize::new(a as usize)), - tree_chunk_size: self.tree_chunk_size, - tree_parallel_persistence_buffer: self - .tree_parallel_persistence_buffer + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + chunk_size: this.chunk_size, + parallel_persistence_buffer: this.parallel_persistence_buffer.map(|a| a.get() as u64), + } + } +} + +impl ProtoRepr for proto::Postgres { + type Type = Postgres; + + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + max_concurrency: self + .max_concurrency .and_then(|a| NonZeroUsize::new(a as usize)), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + max_concurrency: this.max_concurrency.map(|a| a.get() as u64), + } + } +} + +impl ProtoRepr for proto::SnapshotRecovery { + type Type = SnapshotRecoveryConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + enabled: self.enabled.unwrap_or_default(), + tree: read_optional_repr(&self.tree) + .context("tree")? + .unwrap_or_default(), + postgres: read_optional_repr(&self.postgres) + .context("postgres")? + .unwrap_or_default(), l1_batch: self.l1_batch.map(L1BatchNumber), }) } fn build(this: &Self::Type) -> Self { + let tree = if this.tree == Tree::default() { + None + } else { + Some(this.tree.clone()) + }; + let postgres = if this.postgres == Postgres::default() { + None + } else { + Some(this.postgres.clone()) + }; Self { enabled: Some(this.enabled), - postgres_max_concurrency: this.postgres_max_concurrency.map(|a| a.get() as u64), - tree_chunk_size: this.tree_chunk_size, + postgres: postgres.as_ref().map(ProtoRepr::build), + tree: tree.as_ref().map(ProtoRepr::build), l1_batch: this.l1_batch.map(|a| a.0), - tree_parallel_persistence_buffer: this - .tree_parallel_persistence_buffer - .map(|a| a.get() as u64), } } } diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index ce1fbf56cb6b..979f74a26d7e 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -34,7 +34,7 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { } pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { - let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; + let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; decode_yaml_repr::(&yaml) } From 3677fb8ef15c26d6284711a9846d10ed276daf9d Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 15:15:24 +0400 Subject: [PATCH 25/35] Fix tets Signed-off-by: Danil --- etc/env/file_based/general.yaml | 7 +++++-- prover/config/src/lib.rs | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 3ceda661cff5..5e2c138f5a54 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -330,8 +330,11 @@ protective_reads_writer: snapshot_recovery: enabled: true - postgres_max_concurrency: 10 - tree_chunk_size: 200000 + postgres: + max_concurrency: 10 + tree: + chunk_size: 200000 + parallel_persistence_buffer: 1 pruning: enabled: true chunk_size: 10 diff --git a/prover/config/src/lib.rs b/prover/config/src/lib.rs index 8614f1677bda..3a913ab6ef06 100644 --- a/prover/config/src/lib.rs +++ b/prover/config/src/lib.rs @@ -49,6 +49,9 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + commitment_generator: None, + pruning: None, + snapshot_recovery: None, }) } From a1bf8ed9d6a0712f7a1853651066d9f177e46adb Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 15:52:12 +0400 Subject: [PATCH 26/35] Fix observability Signed-off-by: Danil --- core/bin/external_node/src/config/observability.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/bin/external_node/src/config/observability.rs b/core/bin/external_node/src/config/observability.rs index efec5a8a0d87..b66e471f0087 100644 --- a/core/bin/external_node/src/config/observability.rs +++ b/core/bin/external_node/src/config/observability.rs @@ -121,11 +121,11 @@ impl ObservabilityENConfig { (None, None, LogFormat::default(), None) }; let (prometheus_port, prometheus_pushgateway_url, prometheus_push_interval_ms) = - if let Some(api) = general_config.api_config.as_ref() { + if let Some(prometheus) = general_config.prometheus_config.as_ref() { ( - Some(api.prometheus.listener_port), - Some(api.prometheus.pushgateway_url.clone()), - api.prometheus.push_interval_ms.unwrap_or_default(), + Some(prometheus.listener_port), + Some(prometheus.pushgateway_url.clone()), + prometheus.push_interval_ms.unwrap_or_default(), ) } else { (None, None, 0) From 9bb522fdfe74e489c2388ce3dd466e5452a15a82 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 18:32:00 +0400 Subject: [PATCH 27/35] Set more requires Signed-off-by: Danil --- core/bin/external_node/src/main.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 931cd2b271eb..82d0775a3eed 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -703,13 +703,17 @@ struct Cli { #[arg(long, default_value = "all")] components: ComponentsToRun, /// Path to the yaml config. If set, it will be used instead of env vars. - #[arg(long, requires = "secrets_path")] + #[arg( + long, + requires = "secrets_path", + requires = "external_node_config_path" + )] config_path: Option, /// Path to the yaml with secrets. If set, it will be used instead of env vars. - #[arg(long, requires = "external_node_config_path")] + #[arg(long, requires = "config_path", requires = "external_node_config_path")] secrets_path: Option, /// Path to the yaml with genesis. If set, it will be used instead of env vars. - #[arg(long, requires = "config_path")] + #[arg(long, requires = "config_path", requires = "secrets_path")] external_node_config_path: Option, /// Path to the yaml with consensus. consensus_path: Option, From 4d38eacd77c879514a291baf076c6c79bde31f00 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 11:46:50 +0200 Subject: [PATCH 28/35] Some nit fixes Signed-off-by: Danil --- core/lib/config/src/configs/pruning.rs | 3 +-- core/lib/config/src/configs/snapshot_recovery.rs | 8 ++++---- .../src/proto/config/snapshot_recovery.proto | 1 - core/lib/protobuf_config/src/snapshot_recovery.rs | 10 +++++----- 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/core/lib/config/src/configs/pruning.rs b/core/lib/config/src/configs/pruning.rs index f1b3a8e74558..d2a5b0e5e9df 100644 --- a/core/lib/config/src/configs/pruning.rs +++ b/core/lib/config/src/configs/pruning.rs @@ -5,8 +5,7 @@ use serde::Deserialize; #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct PruningConfig { pub enabled: bool, - /// Chunk size for multi-get operations. Can speed up loading data for the Merkle tree on some environments, - /// but the effects vary wildly depending on the setup (e.g., the filesystem used). + /// Number of L1 batches pruned at a time. pub chunk_size: Option, /// Delta between soft- and hard-removing data from Postgres. Should be reasonably large (order of 60 seconds). /// The default value is 60 seconds. diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index d2cf0da190d2..ef2ba49011dd 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -4,7 +4,7 @@ use serde::Deserialize; use zksync_basic_types::L1BatchNumber; #[derive(Debug, Clone, PartialEq, Deserialize, Default)] -pub struct Tree { +pub struct TreeRecoveryConfig { /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. /// Reasonable values are order of 100,000 (meaning an iteration takes several seconds). /// @@ -19,7 +19,7 @@ pub struct Tree { } #[derive(Debug, Clone, PartialEq, Deserialize, Default)] -pub struct Postgres { +pub struct PostgresRecoveryConfig { /// Maximum concurrency factor for the concurrent parts of snapshot recovery for Postgres. It may be useful to /// reduce this factor to about 5 if snapshot recovery overloads I/O capacity of the node. Conversely, /// if I/O capacity of your infra is high, you may increase concurrency to speed up Postgres recovery. @@ -36,6 +36,6 @@ pub struct SnapshotRecoveryConfig { pub enabled: bool, /// L1 batch number of the snapshot to use during recovery. Specifying this parameter is mostly useful for testing. pub l1_batch: Option, - pub tree: Tree, - pub postgres: Postgres, + pub tree: TreeRecoveryConfig, + pub postgres: PostgresRecoveryConfig, } diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index 5f217723f8a7..5faf0399ba69 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -17,4 +17,3 @@ message SnapshotRecovery { optional Tree tree = 3; optional uint32 l1_batch = 4; } - diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 7a84d1a158c6..237f6057d794 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -3,7 +3,7 @@ use std::num::NonZeroUsize; use anyhow::Context; use zksync_basic_types::L1BatchNumber; use zksync_config::configs::{ - snapshot_recovery::{Postgres, Tree}, + snapshot_recovery::{PostgresRecoveryConfig, TreeRecoveryConfig}, SnapshotRecoveryConfig, }; use zksync_protobuf::ProtoRepr; @@ -11,7 +11,7 @@ use zksync_protobuf::ProtoRepr; use crate::{proto::snapshot_recovery as proto, read_optional_repr}; impl ProtoRepr for proto::Tree { - type Type = Tree; + type Type = TreeRecoveryConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { @@ -31,7 +31,7 @@ impl ProtoRepr for proto::Tree { } impl ProtoRepr for proto::Postgres { - type Type = Postgres; + type Type = PostgresRecoveryConfig; fn read(&self) -> anyhow::Result { Ok(Self::Type { @@ -65,12 +65,12 @@ impl ProtoRepr for proto::SnapshotRecovery { } fn build(this: &Self::Type) -> Self { - let tree = if this.tree == Tree::default() { + let tree = if this.tree == TreeRecoveryConfig::default() { None } else { Some(this.tree.clone()) }; - let postgres = if this.postgres == Postgres::default() { + let postgres = if this.postgres == PostgresRecoveryConfig::default() { None } else { Some(this.postgres.clone()) From c99be45334f07febb34bb968d1677fe2841341f7 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 21:31:59 +0200 Subject: [PATCH 29/35] Switch from serde to strum for namespaces Signed-off-by: Danil --- Cargo.lock | 2 ++ Cargo.toml | 1 + core/bin/external_node/src/config/mod.rs | 2 +- core/bin/zksync_server/src/node_builder.rs | 5 ++++- core/node/api_server/Cargo.toml | 2 ++ core/node/api_server/src/web3/mod.rs | 3 ++- prover/Cargo.lock | 2 ++ 7 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a99150fe01c0..f123180cdf50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8807,6 +8807,8 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "strum", + "strum_macros", "test-casing", "thiserror", "thread_local", diff --git a/Cargo.toml b/Cargo.toml index 5d9f6adf37ad..216030b56a2c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,6 +159,7 @@ sqlx = "0.7.3" static_assertions = "1.1" structopt = "0.3.20" strum = "0.24" +strum_macros = "0.24" tempfile = "3.0.2" test-casing = "0.1.2" test-log = "0.2.15" diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 10e32c17ca69..e83d3d679b29 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -444,7 +444,7 @@ pub(crate) struct OptionalENConfig { impl OptionalENConfig { fn from_configs(general_config: &GeneralConfig, enconfig: &ENConfig) -> anyhow::Result { let api_namespaces = load_config!(general_config.api_config, web3_json_rpc.api_namespaces) - .map(|a: Vec| serde_json::from_str(&a.join(","))) + .map(|a: Vec| a.iter().map(|a| a.parse()).collect::>()) .transpose()?; Ok(OptionalENConfig { diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 374126176b52..0206f7c3a61f 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -293,7 +293,10 @@ impl MainNodeBuilder { let with_debug_namespace = state_keeper_config.save_call_traces; let mut namespaces = if let Some(namespaces) = &rpc_config.api_namespaces { - serde_json::from_str(&namespaces.join(","))? + namespaces + .iter() + .map(|a| a.parse()) + .collect::>()? } else { Namespace::DEFAULT.to_vec() }; diff --git a/core/node/api_server/Cargo.toml b/core/node/api_server/Cargo.toml index b826a8b40f21..6f0fad424a22 100644 --- a/core/node/api_server/Cargo.toml +++ b/core/node/api_server/Cargo.toml @@ -48,6 +48,8 @@ pin-project-lite.workspace = true hex.workspace = true http.workspace = true tower.workspace = true +strum.workspace = true +strum_macros.workspace = true tower-http = { workspace = true, features = ["cors", "metrics"] } lru.workspace = true diff --git a/core/node/api_server/src/web3/mod.rs b/core/node/api_server/src/web3/mod.rs index b86666ea6868..32ca1816254d 100644 --- a/core/node/api_server/src/web3/mod.rs +++ b/core/node/api_server/src/web3/mod.rs @@ -86,8 +86,9 @@ enum ApiTransport { Http(SocketAddr), } -#[derive(Debug, Deserialize, Clone, PartialEq)] +#[derive(Debug, Deserialize, Clone, PartialEq, strum_macros::EnumString)] #[serde(rename_all = "lowercase")] +#[strum(serialize_all = "lowercase")] pub enum Namespace { Eth, Net, diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 44c2a8b8395f..ee9933ade209 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -9014,6 +9014,8 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", + "strum", + "strum_macros", "thiserror", "thread_local", "tokio", From 9fb316435e1db498e2dffb8c7aea2d5d6843176d Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 19 Jun 2024 17:51:52 +0200 Subject: [PATCH 30/35] Update comments Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 4 ++-- core/lib/config/src/configs/commitment_generator.rs | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e83d3d679b29..6ff22fda24f7 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -436,7 +436,7 @@ pub(crate) struct OptionalENConfig { /// may be temporarily retained for other reasons; e.g., a batch cannot be pruned until it is executed on L1, /// which happens roughly 24 hours after its generation on the mainnet. Thus, in practice this value can specify /// the retention period greater than that implicitly imposed by other criteria (e.g., 7 or 30 days). - /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 1 hour. + /// If set to 0, L1 batches will not be retained based on their timestamp. The default value is 7 days. #[serde(default = "OptionalENConfig::default_pruning_data_retention_sec")] pruning_data_retention_sec: u64, } @@ -794,7 +794,7 @@ impl OptionalENConfig { } fn default_pruning_data_retention_sec() -> u64 { - 3_600 // 1 hour + 3_600 * 24 * 7 // 7 days } fn from_env() -> anyhow::Result { diff --git a/core/lib/config/src/configs/commitment_generator.rs b/core/lib/config/src/configs/commitment_generator.rs index 4a06bfdba63c..9ec4d805b8fe 100644 --- a/core/lib/config/src/configs/commitment_generator.rs +++ b/core/lib/config/src/configs/commitment_generator.rs @@ -4,5 +4,7 @@ use serde::Deserialize; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct CommitmentGeneratorConfig { + /// Maximum degree of parallelism during commitment generation, i.e., the maximum number of L1 batches being processed in parallel. + /// If not specified, commitment generator will use a value roughly equal to the number of CPU cores with some clamping applied. pub max_parallelism: NonZeroU32, } From b04fc651221b82a1b3777695d5280fb546bb1fee Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 20 Jun 2024 14:12:19 +0200 Subject: [PATCH 31/35] Update curve-dalek Signed-off-by: Danil --- Cargo.lock | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dd76c5c65f5a..9480bd3a7a0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1509,16 +1509,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -4387,12 +4386,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "platforms" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" - [[package]] name = "plotters" version = "0.3.5" From 058322fc9e0cf66588ac68ea9feeedb141314bb8 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 12:35:15 +0200 Subject: [PATCH 32/35] Split to experimental and normal config for tree Signed-off-by: Danil --- .../src/proto/config/experimental.proto | 5 ++ .../src/proto/config/snapshot_recovery.proto | 5 +- .../protobuf_config/src/snapshot_recovery.rs | 63 +++++++++++-------- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/core/lib/protobuf_config/src/proto/config/experimental.proto b/core/lib/protobuf_config/src/proto/config/experimental.proto index ddcb4265319d..6f9ec426d8bb 100644 --- a/core/lib/protobuf_config/src/proto/config/experimental.proto +++ b/core/lib/protobuf_config/src/proto/config/experimental.proto @@ -12,3 +12,8 @@ message DB { optional uint64 processing_delay_ms = 4; optional bool include_indices_and_filters_in_block_cache = 5; } + +// Experimental part of the Snapshot recovery configuration. +message SnapshotRecovery { + optional uint64 tree_recovery_parallel_persistence_buffer = 1; +} diff --git a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto index 5faf0399ba69..9eceda12ad86 100644 --- a/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto +++ b/core/lib/protobuf_config/src/proto/config/snapshot_recovery.proto @@ -1,10 +1,11 @@ syntax = "proto3"; +import "zksync/config/object_store.proto"; +import "zksync/config/experimental.proto"; package zksync.config.snapshot_recovery; message Tree { optional uint64 chunk_size = 1; - optional uint64 parallel_persistence_buffer = 2; } message Postgres { @@ -16,4 +17,6 @@ message SnapshotRecovery { optional Postgres postgres = 2; optional Tree tree = 3; optional uint32 l1_batch = 4; + optional config.object_store.ObjectStore object_store = 5; + optional experimental.SnapshotRecovery experimental = 6; } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index 237f6057d794..ea16d1b158b4 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -10,26 +10,6 @@ use zksync_protobuf::ProtoRepr; use crate::{proto::snapshot_recovery as proto, read_optional_repr}; -impl ProtoRepr for proto::Tree { - type Type = TreeRecoveryConfig; - - fn read(&self) -> anyhow::Result { - Ok(Self::Type { - chunk_size: self.chunk_size, - parallel_persistence_buffer: self - .parallel_persistence_buffer - .and_then(|a| NonZeroUsize::new(a as usize)), - }) - } - - fn build(this: &Self::Type) -> Self { - Self { - chunk_size: this.chunk_size, - parallel_persistence_buffer: this.parallel_persistence_buffer.map(|a| a.get() as u64), - } - } -} - impl ProtoRepr for proto::Postgres { type Type = PostgresRecoveryConfig; @@ -52,11 +32,29 @@ impl ProtoRepr for proto::SnapshotRecovery { type Type = SnapshotRecoveryConfig; fn read(&self) -> anyhow::Result { + let tree = self + .tree + .as_ref() + .map(|tree| { + let chunk_size = tree.chunk_size; + let parallel_persistence_buffer = self + .experimental + .as_ref() + .and_then(|a| { + a.tree_recovery_parallel_persistence_buffer + .map(|a| NonZeroUsize::new(a as usize)) + }) + .flatten(); + TreeRecoveryConfig { + chunk_size, + parallel_persistence_buffer, + } + }) + .unwrap_or_default(); + Ok(Self::Type { enabled: self.enabled.unwrap_or_default(), - tree: read_optional_repr(&self.tree) - .context("tree")? - .unwrap_or_default(), + tree, postgres: read_optional_repr(&self.postgres) .context("postgres")? .unwrap_or_default(), @@ -65,10 +63,20 @@ impl ProtoRepr for proto::SnapshotRecovery { } fn build(this: &Self::Type) -> Self { - let tree = if this.tree == TreeRecoveryConfig::default() { - None + let (tree, experimental) = if this.tree == TreeRecoveryConfig::default() { + (None, None) } else { - Some(this.tree.clone()) + ( + Some(proto::Tree { + chunk_size: this.tree.chunk_size, + }), + Some(crate::proto::experimental::SnapshotRecovery { + tree_recovery_parallel_persistence_buffer: this + .tree + .parallel_persistence_buffer + .map(|a| a.get() as u64), + }), + ) }; let postgres = if this.postgres == PostgresRecoveryConfig::default() { None @@ -78,7 +86,8 @@ impl ProtoRepr for proto::SnapshotRecovery { Self { enabled: Some(this.enabled), postgres: postgres.as_ref().map(ProtoRepr::build), - tree: tree.as_ref().map(ProtoRepr::build), + tree, + experimental, l1_batch: this.l1_batch.map(|a| a.0), } } From 3fd0e5e559701dada39ccd52b03d77b24e9896e0 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 17:26:55 +0200 Subject: [PATCH 33/35] Fix object store Signed-off-by: Danil --- core/lib/config/src/configs/snapshot_recovery.rs | 3 +++ core/lib/protobuf_config/src/snapshot_recovery.rs | 2 ++ 2 files changed, 5 insertions(+) diff --git a/core/lib/config/src/configs/snapshot_recovery.rs b/core/lib/config/src/configs/snapshot_recovery.rs index ef2ba49011dd..ba26583a8a63 100644 --- a/core/lib/config/src/configs/snapshot_recovery.rs +++ b/core/lib/config/src/configs/snapshot_recovery.rs @@ -3,6 +3,8 @@ use std::num::NonZeroUsize; use serde::Deserialize; use zksync_basic_types::L1BatchNumber; +use crate::ObjectStoreConfig; + #[derive(Debug, Clone, PartialEq, Deserialize, Default)] pub struct TreeRecoveryConfig { /// Approximate chunk size (measured in the number of entries) to recover in a single iteration. @@ -38,4 +40,5 @@ pub struct SnapshotRecoveryConfig { pub l1_batch: Option, pub tree: TreeRecoveryConfig, pub postgres: PostgresRecoveryConfig, + pub object_store: Option, } diff --git a/core/lib/protobuf_config/src/snapshot_recovery.rs b/core/lib/protobuf_config/src/snapshot_recovery.rs index ea16d1b158b4..4023cbb0c097 100644 --- a/core/lib/protobuf_config/src/snapshot_recovery.rs +++ b/core/lib/protobuf_config/src/snapshot_recovery.rs @@ -59,6 +59,7 @@ impl ProtoRepr for proto::SnapshotRecovery { .context("postgres")? .unwrap_or_default(), l1_batch: self.l1_batch.map(L1BatchNumber), + object_store: read_optional_repr(&self.object_store).context("object store")?, }) } @@ -89,6 +90,7 @@ impl ProtoRepr for proto::SnapshotRecovery { tree, experimental, l1_batch: this.l1_batch.map(|a| a.0), + object_store: this.object_store.as_ref().map(ProtoRepr::build), } } } From 506bbdcfbb609a209f2965763e7cdb00bd660430 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 17:32:57 +0200 Subject: [PATCH 34/35] Use derive from strum Signed-off-by: Danil --- Cargo.lock | 1 - Cargo.toml | 1 - core/node/api_server/Cargo.toml | 3 +-- core/node/api_server/src/web3/mod.rs | 2 +- etc/env/file_based/general.yaml | 3 ++- 5 files changed, 4 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9480bd3a7a0c..6800b5f5f8f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8640,7 +8640,6 @@ dependencies = [ "serde", "serde_json", "strum", - "strum_macros", "test-casing", "thiserror", "thread_local", diff --git a/Cargo.toml b/Cargo.toml index 216030b56a2c..5d9f6adf37ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -159,7 +159,6 @@ sqlx = "0.7.3" static_assertions = "1.1" structopt = "0.3.20" strum = "0.24" -strum_macros = "0.24" tempfile = "3.0.2" test-casing = "0.1.2" test-log = "0.2.15" diff --git a/core/node/api_server/Cargo.toml b/core/node/api_server/Cargo.toml index 6f0fad424a22..da872ceb41ea 100644 --- a/core/node/api_server/Cargo.toml +++ b/core/node/api_server/Cargo.toml @@ -48,8 +48,7 @@ pin-project-lite.workspace = true hex.workspace = true http.workspace = true tower.workspace = true -strum.workspace = true -strum_macros.workspace = true +strum = { workspace = true, features = ["derive"] } tower-http = { workspace = true, features = ["cors", "metrics"] } lru.workspace = true diff --git a/core/node/api_server/src/web3/mod.rs b/core/node/api_server/src/web3/mod.rs index 32ca1816254d..7b2dec7abb35 100644 --- a/core/node/api_server/src/web3/mod.rs +++ b/core/node/api_server/src/web3/mod.rs @@ -86,7 +86,7 @@ enum ApiTransport { Http(SocketAddr), } -#[derive(Debug, Deserialize, Clone, PartialEq, strum_macros::EnumString)] +#[derive(Debug, Deserialize, Clone, PartialEq, strum::EnumString)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] pub enum Namespace { diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 522fde93a1f9..c03c36f28da2 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -339,7 +339,8 @@ snapshot_recovery: max_concurrency: 10 tree: chunk_size: 200000 - parallel_persistence_buffer: 1 + experimental: + tree_recovery_parallel_persistence_buffer: 1 pruning: enabled: true chunk_size: 10 From ecbec5ad30643cc5e6b3b6e96c074b18fb2e924e Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 25 Jun 2024 12:46:21 +0200 Subject: [PATCH 35/35] Fix defaults Signed-off-by: Danil --- core/bin/external_node/src/config/mod.rs | 11 +++++++---- core/bin/external_node/src/main.rs | 2 +- core/lib/config/src/configs/experimental.rs | 5 +++-- core/lib/config/src/testonly.rs | 2 +- core/lib/protobuf_config/src/experimental.rs | 6 ++++-- .../src/temp_config_store/mod.rs | 2 +- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 6f855d905fd6..35750cfa4e7d 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -577,7 +577,7 @@ impl OptionalENConfig { ), merkle_tree_block_cache_size_mb: load_config_or_default!( general_config.db_config, - experimental.state_keeper_db_block_cache_capacity_mb, + merkle_tree.block_cache_size_mb, default_merkle_tree_block_cache_size_mb ), merkle_tree_memtable_capacity_mb: load_config_or_default!( @@ -637,8 +637,8 @@ impl OptionalENConfig { protective_reads_persistence_enabled: general_config .db_config .as_ref() - .map(|a| a.experimental.reads_persistence_enabled) - .unwrap_or_default(), + .map(|a| a.experimental.protective_reads_persistence_enabled) + .unwrap_or(true), merkle_tree_processing_delay_ms: load_config_or_default!( general_config.db_config, experimental.processing_delay_ms, @@ -936,7 +936,10 @@ impl RequiredENConfig { .api_config .as_ref() .context("Api config is required")?; - let db_config = general.db_config.as_ref().context("Database config")?; + let db_config = general + .db_config + .as_ref() + .context("Database config is required")?; Ok(RequiredENConfig { l1_chain_id: en_config.l1_chain_id, l2_chain_id: en_config.l2_chain_id, diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 8922bd68f3f3..9a1a861fb754 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -714,7 +714,7 @@ struct Cli { /// Path to the yaml with secrets. If set, it will be used instead of env vars. #[arg(long, requires = "config_path", requires = "external_node_config_path")] secrets_path: Option, - /// Path to the yaml with genesis. If set, it will be used instead of env vars. + /// Path to the yaml with external node specific configuration. If set, it will be used instead of env vars. #[arg(long, requires = "config_path", requires = "secrets_path")] external_node_config_path: Option, /// Path to the yaml with consensus. diff --git a/core/lib/config/src/configs/experimental.rs b/core/lib/config/src/configs/experimental.rs index 46d93cdf18f6..e362715d3d4a 100644 --- a/core/lib/config/src/configs/experimental.rs +++ b/core/lib/config/src/configs/experimental.rs @@ -17,7 +17,7 @@ pub struct ExperimentalDBConfig { /// (presumably, to participate in L1 batch proving). /// By default, set to `true` as a temporary safety measure. #[serde(default = "ExperimentalDBConfig::default_protective_reads_persistence_enabled")] - pub reads_persistence_enabled: bool, + pub protective_reads_persistence_enabled: bool, // Merkle tree config /// Processing delay between processing L1 batches in the Merkle tree. #[serde(default = "ExperimentalDBConfig::default_merkle_tree_processing_delay_ms")] @@ -35,7 +35,8 @@ impl Default for ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: Self::default_state_keeper_db_block_cache_capacity_mb(), state_keeper_db_max_open_files: None, - reads_persistence_enabled: Self::default_protective_reads_persistence_enabled(), + protective_reads_persistence_enabled: + Self::default_protective_reads_persistence_enabled(), processing_delay_ms: Self::default_merkle_tree_processing_delay_ms(), include_indices_and_filters_in_block_cache: false, } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index c30551b54570..fd1059b0f32f 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -284,7 +284,7 @@ impl Distribution for EncodeDist { configs::ExperimentalDBConfig { state_keeper_db_block_cache_capacity_mb: self.sample(rng), state_keeper_db_max_open_files: self.sample(rng), - reads_persistence_enabled: self.sample(rng), + protective_reads_persistence_enabled: self.sample(rng), processing_delay_ms: self.sample(rng), include_indices_and_filters_in_block_cache: self.sample(rng), } diff --git a/core/lib/protobuf_config/src/experimental.rs b/core/lib/protobuf_config/src/experimental.rs index e3d864c51c42..8d92f3ef87a8 100644 --- a/core/lib/protobuf_config/src/experimental.rs +++ b/core/lib/protobuf_config/src/experimental.rs @@ -21,7 +21,9 @@ impl ProtoRepr for proto::Db { .map(|count| NonZeroU32::new(count).context("cannot be 0")) .transpose() .context("state_keeper_db_max_open_files")?, - reads_persistence_enabled: self.reads_persistence_enabled.unwrap_or_default(), + protective_reads_persistence_enabled: self + .reads_persistence_enabled + .unwrap_or_default(), processing_delay_ms: self.processing_delay_ms.unwrap_or_default(), include_indices_and_filters_in_block_cache: self .include_indices_and_filters_in_block_cache @@ -39,7 +41,7 @@ impl ProtoRepr for proto::Db { state_keeper_db_max_open_files: this .state_keeper_db_max_open_files .map(NonZeroU32::get), - reads_persistence_enabled: Some(this.reads_persistence_enabled), + reads_persistence_enabled: Some(this.protective_reads_persistence_enabled), processing_delay_ms: Some(this.processing_delay_ms), include_indices_and_filters_in_block_cache: Some( this.include_indices_and_filters_in_block_cache, diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 8764025ef26e..60a610c359f8 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -28,7 +28,7 @@ pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { } pub fn read_yaml_repr(path_buf: PathBuf) -> anyhow::Result { - let yaml = std::fs::read_to_string(path_buf).context("failed decoding YAML config")?; + let yaml = std::fs::read_to_string(path_buf).context("failed reading YAML config")?; decode_yaml_repr::(&yaml) }