diff --git a/Cargo.lock b/Cargo.lock index 29e9b3be3b9f..953001c820a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -924,6 +924,7 @@ dependencies = [ "zksync_config", "zksync_core", "zksync_dal", + "zksync_env_config", "zksync_types", ] @@ -3811,6 +3812,7 @@ dependencies = [ "tracing", "vlog", "zksync_config", + "zksync_env_config", "zksync_merkle_tree", "zksync_storage", "zksync_types", @@ -5471,6 +5473,7 @@ dependencies = [ "clap 4.4.6", "tempfile", "zksync_config", + "zksync_env_config", "zksync_storage", ] @@ -7935,17 +7938,10 @@ name = "zksync_config" version = "0.1.0" dependencies = [ "anyhow", - "bigdecimal", "envy", - "hex", - "num 0.3.1", - "once_cell", "serde", - "serde_json", - "url", "zksync_basic_types", "zksync_contracts", - "zksync_utils", ] [[package]] @@ -7973,6 +7969,7 @@ dependencies = [ "zksync_config", "zksync_contracts", "zksync_dal", + "zksync_env_config", "zksync_queued_job_processor", "zksync_types", "zksync_utils", @@ -8101,6 +8098,17 @@ dependencies = [ "zksync_utils", ] +[[package]] +name = "zksync_env_config" +version = "0.1.0" +dependencies = [ + "anyhow", + "envy", + "serde", + "zksync_basic_types", + "zksync_config", +] + [[package]] name = "zksync_eth_client" version = "0.1.0" @@ -8287,6 +8295,7 @@ dependencies = [ "vlog", "zksync_config", "zksync_core", + "zksync_env_config", "zksync_storage", "zksync_types", "zksync_utils", diff --git a/Cargo.toml b/Cargo.toml index a02652c3f617..75a4c7237d2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "core/lib/circuit_breaker", "core/lib/commitment_utils", "core/lib/dal", + "core/lib/env_config", "core/lib/eth_client", "core/lib/eth_signer", "core/lib/mempool", diff --git a/core/bin/block_reverter/Cargo.toml b/core/bin/block_reverter/Cargo.toml index 33147ad1cac1..634132f33058 100644 --- a/core/bin/block_reverter/Cargo.toml +++ b/core/bin/block_reverter/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_dal = { path = "../../lib/dal" } zksync_types = { path = "../../lib/types" } zksync_core = { path = "../../lib/zksync_core" } diff --git a/core/bin/block_reverter/src/main.rs b/core/bin/block_reverter/src/main.rs index 971dfbeb8d15..443f6fe634b5 100644 --- a/core/bin/block_reverter/src/main.rs +++ b/core/bin/block_reverter/src/main.rs @@ -4,6 +4,7 @@ use tokio::io::{self, AsyncReadExt}; use zksync_config::{ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig}; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::FromEnv; use zksync_types::{L1BatchNumber, U256}; use zksync_core::block_reverter::{ diff --git a/core/bin/contract-verifier/Cargo.toml b/core/bin/contract-verifier/Cargo.toml index e86d24931bc8..1a22c1825b51 100644 --- a/core/bin/contract-verifier/Cargo.toml +++ b/core/bin/contract-verifier/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_types = { path = "../../lib/types" } zksync_dal = { path = "../../lib/dal" } +zksync_env_config = { path = "../../lib/env_config" } zksync_config = { path = "../../lib/config" } zksync_contracts = { path = "../../lib/contracts" } zksync_queued_job_processor = { path = "../../lib/queued_job_processor" } diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 29696e58896d..a5c24450f6b6 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -4,6 +4,7 @@ use anyhow::Context as _; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig}; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; diff --git a/core/bin/contract-verifier/src/verifier.rs b/core/bin/contract-verifier/src/verifier.rs index 97c42899254b..03e94dde75ce 100644 --- a/core/bin/contract-verifier/src/verifier.rs +++ b/core/bin/contract-verifier/src/verifier.rs @@ -12,6 +12,7 @@ use tokio::time; use zksync_config::ContractVerifierConfig; use zksync_dal::{ConnectionPool, StorageProcessor}; +use zksync_env_config::FromEnv; use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ contract_verification_api::{ diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 652ddd6e33ca..2c5666ee0e03 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -3,8 +3,7 @@ use serde::Deserialize; use std::{env, time::Duration}; use url::Url; -use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber, H256}; -use zksync_contracts::BaseSystemContractsHashes; +use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber}; use zksync_core::api_server::{ tx_sender::TxSenderConfig, web3::state::InternalApiConfig, web3::Namespace, }; @@ -32,9 +31,6 @@ pub struct RemoteENConfig { pub l2_chain_id: L2ChainId, pub l1_chain_id: L1ChainId, - pub default_aa_hash: H256, - pub bootloader_hash: H256, - pub fair_l2_gas_price: u64, } @@ -76,7 +72,6 @@ impl RemoteENConfig { .await .context("Failed to fetch last miniblock header")? .expect("Block is known to exist"); - let base_system_contract_hashes = block_header.base_system_contracts_hashes; Ok(Self { diamond_proxy_addr, @@ -87,18 +82,9 @@ impl RemoteENConfig { l2_weth_bridge_addr: bridges.l2_weth_bridge, l2_chain_id, l1_chain_id, - default_aa_hash: base_system_contract_hashes.default_aa, - bootloader_hash: base_system_contract_hashes.bootloader, fair_l2_gas_price: block_header.l2_fair_gas_price, }) } - - pub fn base_system_contracts_hashes(&self) -> BaseSystemContractsHashes { - BaseSystemContractsHashes { - default_aa: self.default_aa_hash, - bootloader: self.bootloader_hash, - } - } } /// This part of the external node config is completely optional to provide. @@ -509,8 +495,6 @@ impl From for TxSenderConfig { max_nonce_ahead: config.optional.max_nonce_ahead, fair_l2_gas_price: config.remote.fair_l2_gas_price, vm_execution_cache_misses_limit: config.optional.vm_execution_cache_misses_limit, - default_aa: config.remote.default_aa_hash, - bootloader: config.remote.bootloader_hash, // We set these values to the maximum since we don't know the actual values // and they will be enforced by the main node anyway. max_allowed_l2_tx_gas_limit: u32::MAX, diff --git a/core/bin/merkle_tree_consistency_checker/Cargo.toml b/core/bin/merkle_tree_consistency_checker/Cargo.toml index 344170c956af..6bd30586d22d 100644 --- a/core/bin/merkle_tree_consistency_checker/Cargo.toml +++ b/core/bin/merkle_tree_consistency_checker/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_merkle_tree = { path = "../../lib/merkle_tree" } zksync_types = { path = "../../lib/types" } zksync_storage = { path = "../../lib/storage" } diff --git a/core/bin/merkle_tree_consistency_checker/src/main.rs b/core/bin/merkle_tree_consistency_checker/src/main.rs index a4a4aeec0bf6..b132bda87fa0 100644 --- a/core/bin/merkle_tree_consistency_checker/src/main.rs +++ b/core/bin/merkle_tree_consistency_checker/src/main.rs @@ -4,6 +4,7 @@ use clap::Parser; use std::{path::Path, time::Instant}; use zksync_config::DBConfig; +use zksync_env_config::FromEnv; use zksync_merkle_tree::domain::ZkSyncTree; use zksync_storage::RocksDB; use zksync_types::L1BatchNumber; diff --git a/core/bin/rocksdb_util/Cargo.toml b/core/bin/rocksdb_util/Cargo.toml index 5418169609f0..75d1afe19797 100644 --- a/core/bin/rocksdb_util/Cargo.toml +++ b/core/bin/rocksdb_util/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_storage = { path = "../../lib/storage" } anyhow = "1.0" diff --git a/core/bin/rocksdb_util/src/main.rs b/core/bin/rocksdb_util/src/main.rs index a579dfdbfb24..30d3d42e771c 100644 --- a/core/bin/rocksdb_util/src/main.rs +++ b/core/bin/rocksdb_util/src/main.rs @@ -2,6 +2,7 @@ use anyhow::Context as _; use clap::{Parser, Subcommand}; use zksync_config::DBConfig; +use zksync_env_config::FromEnv; use zksync_storage::rocksdb::{ backup::{BackupEngine, BackupEngineOptions, RestoreOptions}, Env, Error, Options, DB, diff --git a/core/bin/zksync_server/Cargo.toml b/core/bin/zksync_server/Cargo.toml index 674736fe98bc..a1de2ef057ad 100644 --- a/core/bin/zksync_server/Cargo.toml +++ b/core/bin/zksync_server/Cargo.toml @@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_config = { path = "../../lib/config" } +zksync_env_config = { path = "../../lib/env_config" } zksync_storage = { path = "../../lib/storage" } zksync_utils = { path = "../../lib/utils" } zksync_types = { path = "../../lib/types" } diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index 4a0bff0cd442..23794f5dafd4 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -2,13 +2,28 @@ use anyhow::Context as _; use clap::Parser; use std::{str::FromStr, time::Duration}; -use zksync_config::configs::chain::NetworkConfig; -use zksync_config::{ContractsConfig, ETHClientConfig, ETHSenderConfig}; +use zksync_config::{ + configs::{ + api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + house_keeper::HouseKeeperConfig, + FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, + ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, + }, + ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig, + FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs, +}; + +use zksync_core::temp_config_store::TempConfigStore; use zksync_core::{ genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler, Component, Components, }; +use zksync_env_config::FromEnv; use zksync_storage::RocksDB; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -102,9 +117,42 @@ async fn main() -> anyhow::Result<()> { [Component::WitnessGenerator(Some(_), _)] ); + // TODO (QIT-22): Only deserialize configs on demand. + // Right now, we are trying to deserialize all the configs that may be needed by `zksync_core`. + // "May" is the key word here, since some configs are only used by certain component configuration, + // hence we are using `Option`s. + let configs = TempConfigStore { + health_check_config: HealthCheckConfig::from_env().ok(), + merkle_tree_api_config: MerkleTreeApiConfig::from_env().ok(), + web3_json_rpc_config: Web3JsonRpcConfig::from_env().ok(), + circuit_breaker_config: CircuitBreakerConfig::from_env().ok(), + mempool_config: MempoolConfig::from_env().ok(), + network_config: NetworkConfig::from_env().ok(), + operations_manager_config: OperationsManagerConfig::from_env().ok(), + state_keeper_config: StateKeeperConfig::from_env().ok(), + house_keeper_config: HouseKeeperConfig::from_env().ok(), + fri_proof_compressor_config: FriProofCompressorConfig::from_env().ok(), + fri_prover_config: FriProverConfig::from_env().ok(), + fri_witness_generator_config: FriWitnessGeneratorConfig::from_env().ok(), + prometheus_config: PrometheusConfig::from_env().ok(), + proof_data_handler_config: ProofDataHandlerConfig::from_env().ok(), + prover_group_config: ProverGroupConfig::from_env().ok(), + witness_generator_config: WitnessGeneratorConfig::from_env().ok(), + api_config: ApiConfig::from_env().ok(), + contracts_config: ContractsConfig::from_env().ok(), + db_config: DBConfig::from_env().ok(), + eth_client_config: ETHClientConfig::from_env().ok(), + eth_sender_config: ETHSenderConfig::from_env().ok(), + eth_watch_config: ETHWatchConfig::from_env().ok(), + fetcher_config: FetcherConfig::from_env().ok(), + gas_adjuster_config: GasAdjusterConfig::from_env().ok(), + prover_configs: ProverConfigs::from_env().ok(), + object_store_config: ObjectStoreConfig::from_env().ok(), + }; + // Run core actors. let (core_task_handles, stop_sender, cb_receiver, health_check_handle) = - initialize_components(components, is_only_oneshot_witness_generator_task) + initialize_components(&configs, components, is_only_oneshot_witness_generator_task) .await .context("Unable to start Core actors")?; diff --git a/core/lib/config/Cargo.toml b/core/lib/config/Cargo.toml index 948ec9aba56e..6ff3363f895a 100644 --- a/core/lib/config/Cargo.toml +++ b/core/lib/config/Cargo.toml @@ -11,15 +11,8 @@ categories = ["cryptography"] [dependencies] zksync_basic_types = { path = "../../lib/basic_types" } -zksync_utils = { path = "../../lib/utils" } zksync_contracts = { path = "../../lib/contracts" } anyhow = "1.0" -url = "2.1" -num = "0.3.1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" envy = "0.4" -once_cell = "1.13.0" -bigdecimal = "0.2.2" -hex = "0.4" diff --git a/core/lib/config/src/configs/alerts.rs b/core/lib/config/src/configs/alerts.rs index ffdd0dea0441..070ebb970fe8 100644 --- a/core/lib/config/src/configs/alerts.rs +++ b/core/lib/config/src/configs/alerts.rs @@ -1,47 +1,8 @@ -// Built-in uses -// External uses use serde::Deserialize; -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct AlertsConfig { /// List of panics' messages from external crypto code, /// that are sporadic and needed to be handled separately pub sporadic_crypto_errors_substrs: Vec, } - -impl AlertsConfig { - pub fn from_env() -> anyhow::Result { - envy_load("sporadic_crypto_errors_substrs", "ALERTS_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> AlertsConfig { - AlertsConfig { - sporadic_crypto_errors_substrs: vec![ - "EventDestroyErr".to_string(), - "Can't free memory of DeviceBuf".to_string(), - "value: PoisonError".to_string(), - ], - } - } - - #[test] - fn test_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ALERTS_SPORADIC_CRYPTO_ERRORS_SUBSTRS="EventDestroyErr,Can't free memory of DeviceBuf,value: PoisonError" - "#; - lock.set_env(config); - - assert_eq!(AlertsConfig::from_env().unwrap(), expected_config()); - } -} diff --git a/core/lib/config/src/configs/api.rs b/core/lib/config/src/configs/api.rs index e1f79d052260..dc14514aec45 100644 --- a/core/lib/config/src/configs/api.rs +++ b/core/lib/config/src/configs/api.rs @@ -1,9 +1,7 @@ -use anyhow::Context as _; use serde::Deserialize; use std::{net::SocketAddr, time::Duration}; -use super::envy_load; pub use crate::configs::PrometheusConfig; use zksync_basic_types::H256; @@ -22,19 +20,6 @@ pub struct ApiConfig { pub merkle_tree: MerkleTreeApiConfig, } -impl ApiConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - web3_json_rpc: Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig")?, - contract_verification: ContractVerificationApiConfig::from_env() - .context("ContractVerificationApiConfig")?, - prometheus: PrometheusConfig::from_env().context("PrometheusConfig")?, - healthcheck: HealthCheckConfig::from_env().context("HealthCheckConfig")?, - merkle_tree: MerkleTreeApiConfig::from_env().context("MerkleTreeApiConfig")?, - }) - } -} - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct Web3JsonRpcConfig { /// Port to which the HTTP RPC server is listening. @@ -105,8 +90,40 @@ pub struct Web3JsonRpcConfig { } impl Web3JsonRpcConfig { - pub fn from_env() -> anyhow::Result { - envy_load("web3_json_rpc", "API_WEB3_JSON_RPC_") + /// Creates a mock instance of `Web3JsonRpcConfig` to be used in tests. + /// Ports and some fields that may affect execution are set to the same values used by default in + /// the localhost environment. Other fields are set to default values. + pub fn for_tests() -> Self { + Self { + http_port: 3050, + http_url: "http://localhost:3050".into(), + ws_port: 3051, + ws_url: "ws://localhost:3051".into(), + req_entities_limit: Some(10000), + filters_limit: Some(10000), + subscriptions_limit: Some(10000), + pubsub_polling_interval: Some(200), + threads_per_server: 1, + max_nonce_ahead: 50, + gas_price_scale_factor: 1.2, + transactions_per_sec_limit: Default::default(), + request_timeout: Default::default(), + account_pks: Default::default(), + estimate_gas_scale_factor: 1.2, + estimate_gas_acceptable_overestimation: 1000, + max_tx_size: 1000000, + vm_execution_cache_misses_limit: Default::default(), + vm_concurrency_limit: Default::default(), + factory_deps_cache_size_mb: Default::default(), + initial_writes_cache_size_mb: Default::default(), + latest_values_cache_size_mb: Default::default(), + http_threads: Default::default(), + ws_threads: Default::default(), + fee_history_limit: Default::default(), + max_batch_request_size: Default::default(), + max_response_body_size_mb: Default::default(), + websocket_requests_per_minute_limit: Default::default(), + } } pub fn http_bind_addr(&self) -> SocketAddr { @@ -197,10 +214,6 @@ pub struct HealthCheckConfig { } impl HealthCheckConfig { - pub fn from_env() -> anyhow::Result { - envy_load("healthcheck", "API_HEALTHCHECK_") - } - pub fn bind_addr(&self) -> SocketAddr { SocketAddr::new("0.0.0.0".parse().unwrap(), self.port) } @@ -220,10 +233,6 @@ impl ContractVerificationApiConfig { pub fn bind_addr(&self) -> SocketAddr { SocketAddr::new("0.0.0.0".parse().unwrap(), self.port) } - - pub fn from_env() -> anyhow::Result { - envy_load("contract_verification", "API_CONTRACT_VERIFICATION_") - } } /// Configuration for the Merkle tree API. @@ -238,131 +247,4 @@ impl MerkleTreeApiConfig { const fn default_port() -> u16 { 3_072 } - - /// Loads configuration from env variables. - pub fn from_env() -> anyhow::Result { - envy_load("merkle_tree_api", "API_MERKLE_TREE_") - } -} - -#[cfg(test)] -mod tests { - use std::net::IpAddr; - - use super::*; - use crate::configs::test_utils::{hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ApiConfig { - ApiConfig { - web3_json_rpc: Web3JsonRpcConfig { - http_port: 3050, - http_url: "http://127.0.0.1:3050".into(), - ws_port: 3051, - ws_url: "ws://127.0.0.1:3051".into(), - req_entities_limit: Some(10000), - filters_limit: Some(10000), - subscriptions_limit: Some(10000), - pubsub_polling_interval: Some(200), - threads_per_server: 128, - max_nonce_ahead: 5, - transactions_per_sec_limit: Some(1000), - request_timeout: Some(10), - account_pks: Some(vec![ - hash("0x0000000000000000000000000000000000000000000000000000000000000001"), - hash("0x0000000000000000000000000000000000000000000000000000000000000002"), - ]), - estimate_gas_scale_factor: 1.0f64, - gas_price_scale_factor: 1.2, - estimate_gas_acceptable_overestimation: 1000, - max_tx_size: 1000000, - vm_execution_cache_misses_limit: None, - vm_concurrency_limit: Some(512), - factory_deps_cache_size_mb: Some(128), - initial_writes_cache_size_mb: Some(32), - latest_values_cache_size_mb: Some(256), - http_threads: Some(128), - ws_threads: Some(256), - fee_history_limit: Some(100), - max_batch_request_size: Some(200), - max_response_body_size_mb: Some(10), - websocket_requests_per_minute_limit: Some(10), - }, - contract_verification: ContractVerificationApiConfig { - port: 3070, - url: "http://127.0.0.1:3070".into(), - threads_per_server: 128, - }, - prometheus: PrometheusConfig { - listener_port: 3312, - pushgateway_url: "http://127.0.0.1:9091".into(), - push_interval_ms: Some(100), - }, - healthcheck: HealthCheckConfig { port: 8081 }, - merkle_tree: MerkleTreeApiConfig { port: 8082 }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - API_WEB3_JSON_RPC_HTTP_PORT="3050" - API_WEB3_JSON_RPC_HTTP_URL="http://127.0.0.1:3050" - API_WEB3_JSON_RPC_WS_PORT="3051" - API_WEB3_JSON_RPC_WS_URL="ws://127.0.0.1:3051" - API_WEB3_JSON_RPC_REQ_ENTITIES_LIMIT=10000 - API_WEB3_JSON_RPC_FILTERS_LIMIT=10000 - API_WEB3_JSON_RPC_SUBSCRIPTIONS_LIMIT=10000 - API_WEB3_JSON_RPC_PUBSUB_POLLING_INTERVAL=200 - API_WEB3_JSON_RPC_THREADS_PER_SERVER=128 - API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 - API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 - API_WEB3_JSON_RPC_TRANSACTIONS_PER_SEC_LIMIT=1000 - API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 - API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" - API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 - API_WEB3_JSON_RPC_ESTIMATE_GAS_ACCEPTABLE_OVERESTIMATION=1000 - API_WEB3_JSON_RPC_MAX_TX_SIZE=1000000 - API_WEB3_JSON_RPC_VM_CONCURRENCY_LIMIT=512 - API_WEB3_JSON_RPC_FACTORY_DEPS_CACHE_SIZE_MB=128 - API_WEB3_JSON_RPC_INITIAL_WRITES_CACHE_SIZE_MB=32 - API_WEB3_JSON_RPC_LATEST_VALUES_CACHE_SIZE_MB=256 - API_WEB3_JSON_RPC_HTTP_THREADS=128 - API_WEB3_JSON_RPC_WS_THREADS=256 - API_WEB3_JSON_RPC_FEE_HISTORY_LIMIT=100 - API_WEB3_JSON_RPC_MAX_BATCH_REQUEST_SIZE=200 - API_WEB3_JSON_RPC_WEBSOCKET_REQUESTS_PER_MINUTE_LIMIT=10 - API_CONTRACT_VERIFICATION_PORT="3070" - API_CONTRACT_VERIFICATION_URL="http://127.0.0.1:3070" - API_CONTRACT_VERIFICATION_THREADS_PER_SERVER=128 - API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_MB=10 - API_PROMETHEUS_LISTENER_PORT="3312" - API_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - API_PROMETHEUS_PUSH_INTERVAL_MS=100 - API_HEALTHCHECK_PORT=8081 - API_MERKLE_TREE_PORT=8082 - "#; - lock.set_env(config); - - let actual = ApiConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - let bind_broadcast_addr: IpAddr = "0.0.0.0".parse().unwrap(); - - assert_eq!( - config.web3_json_rpc.pubsub_interval(), - Duration::from_millis(200) - ); - assert_eq!( - config.contract_verification.bind_addr(), - SocketAddr::new(bind_broadcast_addr, config.contract_verification.port) - ); - } } diff --git a/core/lib/config/src/configs/chain.rs b/core/lib/config/src/configs/chain.rs index ddf0c85e63e5..95392c8df839 100644 --- a/core/lib/config/src/configs/chain.rs +++ b/core/lib/config/src/configs/chain.rs @@ -1,14 +1,11 @@ /// External uses -use anyhow::Context as _; use serde::Deserialize; +use std::str::FromStr; /// Built-in uses use std::time::Duration; // Local uses use zksync_basic_types::network::Network; -use zksync_basic_types::{Address, L2ChainId, H256}; -use zksync_contracts::BaseSystemContractsHashes; - -use super::envy_load; +use zksync_basic_types::{Address, L2ChainId}; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ChainConfig { @@ -24,20 +21,6 @@ pub struct ChainConfig { pub circuit_breaker: CircuitBreakerConfig, } -impl ChainConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - // TODO rename `eth` to `network` - network: NetworkConfig::from_env().context("NetworkConfig")?, - state_keeper: StateKeeperConfig::from_env().context("StateKeeperConfig")?, - operations_manager: OperationsManagerConfig::from_env() - .context("OperationsManagerConfig")?, - mempool: MempoolConfig::from_env().context("MempoolConfig")?, - circuit_breaker: CircuitBreakerConfig::from_env().context("CircuitBreakerConfig")?, - }) - } -} - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct NetworkConfig { /// Name of the used Ethereum network, e.g. `localhost` or `rinkeby`. @@ -51,8 +34,13 @@ pub struct NetworkConfig { } impl NetworkConfig { - pub fn from_env() -> anyhow::Result { - envy_load("network", "CHAIN_ETH_") + /// Creates a config object suitable for use in unit tests. + pub fn for_tests() -> NetworkConfig { + Self { + network: Network::Localhost, + zksync_network: "localhost".into(), + zksync_network_id: L2ChainId::default(), + } } } @@ -96,9 +84,6 @@ pub struct StateKeeperConfig { /// The price the operator spends on 1 gas of computation in wei. pub fair_l2_gas_price: u64, - pub bootloader_hash: H256, - pub default_aa_hash: H256, - /// Max number of computational gas that validation step is allowed to take. pub validation_computational_gas_limit: u32, pub save_call_traces: bool, @@ -115,14 +100,31 @@ pub struct StateKeeperConfig { } impl StateKeeperConfig { - pub fn from_env() -> anyhow::Result { - envy_load("state_keeper", "CHAIN_STATE_KEEPER_") - } - - pub fn base_system_contracts_hashes(&self) -> BaseSystemContractsHashes { - BaseSystemContractsHashes { - bootloader: self.bootloader_hash, - default_aa: self.default_aa_hash, + /// Creates a config object suitable for use in unit tests. + /// Values mostly repeat the values used in the localhost environment. + pub fn for_tests() -> Self { + Self { + transaction_slots: 250, + block_commit_deadline_ms: 2500, + miniblock_commit_deadline_ms: 1000, + miniblock_seal_queue_capacity: 10, + max_single_tx_gas: 6000000, + max_allowed_l2_tx_gas_limit: 4000000000, + reject_tx_at_geometry_percentage: 0.95, + reject_tx_at_eth_params_percentage: 0.95, + reject_tx_at_gas_percentage: 0.95, + close_block_at_geometry_percentage: 0.95, + close_block_at_eth_params_percentage: 0.95, + close_block_at_gas_percentage: 0.95, + fee_account_addr: Address::from_str("0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7") + .unwrap(), + fair_l2_gas_price: 250000000, + validation_computational_gas_limit: 300000, + save_call_traces: true, + virtual_blocks_interval: 1, + virtual_blocks_per_miniblock: 1, + upload_witness_inputs_to_gcs: false, + enum_index_migration_chunk_size: None, } } @@ -138,10 +140,6 @@ pub struct OperationsManagerConfig { } impl OperationsManagerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("operations_manager", "CHAIN_OPERATIONS_MANAGER_") - } - pub fn delay_interval(&self) -> Duration { Duration::from_millis(self.delay_interval) } @@ -156,10 +154,6 @@ pub struct CircuitBreakerConfig { } impl CircuitBreakerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("circuit_breaker", "CHAIN_CIRCUIT_BREAKER_") - } - pub fn sync_interval(&self) -> Duration { Duration::from_millis(self.sync_interval_ms) } @@ -191,112 +185,4 @@ impl MempoolConfig { pub fn delay_interval(&self) -> Duration { Duration::from_millis(self.delay_interval) } - - pub fn from_env() -> anyhow::Result { - envy_load("mempool", "CHAIN_MEMPOOL_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{addr, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ChainConfig { - ChainConfig { - network: NetworkConfig { - network: "localhost".parse().unwrap(), - zksync_network: "localhost".to_string(), - zksync_network_id: L2ChainId::from(270), - }, - state_keeper: StateKeeperConfig { - transaction_slots: 50, - block_commit_deadline_ms: 2500, - miniblock_commit_deadline_ms: 1000, - miniblock_seal_queue_capacity: 10, - max_single_tx_gas: 1_000_000, - max_allowed_l2_tx_gas_limit: 2_000_000_000, - close_block_at_eth_params_percentage: 0.2, - close_block_at_gas_percentage: 0.8, - close_block_at_geometry_percentage: 0.5, - reject_tx_at_eth_params_percentage: 0.8, - reject_tx_at_geometry_percentage: 0.3, - fee_account_addr: addr("de03a0B5963f75f1C8485B355fF6D30f3093BDE7"), - reject_tx_at_gas_percentage: 0.5, - fair_l2_gas_price: 250000000, - bootloader_hash: H256::from(&[254; 32]), - default_aa_hash: H256::from(&[254; 32]), - validation_computational_gas_limit: 10_000_000, - save_call_traces: false, - virtual_blocks_interval: 1, - virtual_blocks_per_miniblock: 1, - upload_witness_inputs_to_gcs: false, - enum_index_migration_chunk_size: Some(2_000), - }, - operations_manager: OperationsManagerConfig { - delay_interval: 100, - }, - mempool: MempoolConfig { - sync_interval_ms: 10, - sync_batch_size: 1000, - capacity: 1_000_000, - stuck_tx_timeout: 10, - remove_stuck_txs: true, - delay_interval: 100, - }, - circuit_breaker: CircuitBreakerConfig { - sync_interval_ms: 1000, - http_req_max_retry_number: 5, - http_req_retry_interval_sec: 2, - replication_lag_limit_sec: Some(10), - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CHAIN_ETH_NETWORK="localhost" - CHAIN_ETH_ZKSYNC_NETWORK="localhost" - CHAIN_ETH_ZKSYNC_NETWORK_ID=270 - CHAIN_STATE_KEEPER_TRANSACTION_SLOTS="50" - CHAIN_STATE_KEEPER_FEE_ACCOUNT_ADDR="0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7" - CHAIN_STATE_KEEPER_MAX_SINGLE_TX_GAS="1000000" - CHAIN_STATE_KEEPER_MAX_ALLOWED_L2_TX_GAS_LIMIT="2000000000" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GEOMETRY_PERCENTAGE="0.5" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GAS_PERCENTAGE="0.8" - CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_ETH_PARAMS_PERCENTAGE="0.2" - CHAIN_STATE_KEEPER_REJECT_TX_AT_GEOMETRY_PERCENTAGE="0.3" - CHAIN_STATE_KEEPER_REJECT_TX_AT_ETH_PARAMS_PERCENTAGE="0.8" - CHAIN_STATE_KEEPER_REJECT_TX_AT_GAS_PERCENTAGE="0.5" - CHAIN_STATE_KEEPER_BLOCK_COMMIT_DEADLINE_MS="2500" - CHAIN_STATE_KEEPER_MINIBLOCK_COMMIT_DEADLINE_MS="1000" - CHAIN_STATE_KEEPER_MINIBLOCK_SEAL_QUEUE_CAPACITY="10" - CHAIN_STATE_KEEPER_FAIR_L2_GAS_PRICE="250000000" - CHAIN_STATE_KEEPER_BOOTLOADER_HASH="0xfefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe" - CHAIN_STATE_KEEPER_DEFAULT_AA_HASH="0xfefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefefe" - CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT="10000000" - CHAIN_STATE_KEEPER_SAVE_CALL_TRACES="false" - CHAIN_STATE_KEEPER_UPLOAD_WITNESS_INPUTS_TO_GCS="false" - CHAIN_STATE_KEEPER_ENUM_INDEX_MIGRATION_CHUNK_SIZE="2000" - CHAIN_OPERATIONS_MANAGER_DELAY_INTERVAL="100" - CHAIN_MEMPOOL_SYNC_INTERVAL_MS="10" - CHAIN_MEMPOOL_SYNC_BATCH_SIZE="1000" - CHAIN_MEMPOOL_STUCK_TX_TIMEOUT="10" - CHAIN_MEMPOOL_REMOVE_STUCK_TXS="true" - CHAIN_MEMPOOL_DELAY_INTERVAL="100" - CHAIN_MEMPOOL_CAPACITY="1000000" - CHAIN_CIRCUIT_BREAKER_SYNC_INTERVAL_MS="1000" - CHAIN_CIRCUIT_BREAKER_HTTP_REQ_MAX_RETRY_NUMBER="5" - CHAIN_CIRCUIT_BREAKER_HTTP_REQ_RETRY_INTERVAL_SEC="2" - CHAIN_CIRCUIT_BREAKER_REPLICATION_LAG_LIMIT_SEC="10" - "#; - lock.set_env(config); - - let actual = ChainConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } } diff --git a/core/lib/config/src/configs/circuit_synthesizer.rs b/core/lib/config/src/configs/circuit_synthesizer.rs index d336b4f5b18c..8df0f4bbd1a1 100644 --- a/core/lib/config/src/configs/circuit_synthesizer.rs +++ b/core/lib/config/src/configs/circuit_synthesizer.rs @@ -2,8 +2,6 @@ use std::time::Duration; use serde::Deserialize; -use super::envy_load; - /// Configuration for the witness generation #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct CircuitSynthesizerConfig { @@ -26,10 +24,6 @@ pub struct CircuitSynthesizerConfig { } impl CircuitSynthesizerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("circuit_synthesizer", "CIRCUIT_SYNTHESIZER_") - } - pub fn generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -46,45 +40,3 @@ impl CircuitSynthesizerConfig { Duration::from_millis(self.prover_instance_poll_time_in_milli_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> CircuitSynthesizerConfig { - CircuitSynthesizerConfig { - generation_timeout_in_secs: 1000u16, - max_attempts: 2, - gpu_prover_queue_timeout_in_secs: 1000u16, - prover_instance_wait_timeout_in_secs: 1000u16, - prover_instance_poll_time_in_milli_secs: 250u16, - prometheus_listener_port: 3314, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - prover_group_id: 0, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CIRCUIT_SYNTHESIZER_GENERATION_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_MAX_ATTEMPTS=2 - CIRCUIT_SYNTHESIZER_GPU_PROVER_QUEUE_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 - CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 - CIRCUIT_SYNTHESIZER_PROMETHEUS_LISTENER_PORT=3314 - CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSH_INTERVAL_MS=100 - CIRCUIT_SYNTHESIZER_PROVER_GROUP_ID=0 - "#; - lock.set_env(config); - - let actual = CircuitSynthesizerConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/contract_verifier.rs b/core/lib/config/src/configs/contract_verifier.rs index 08f05e8f264d..5c2a1608c8f5 100644 --- a/core/lib/config/src/configs/contract_verifier.rs +++ b/core/lib/config/src/configs/contract_verifier.rs @@ -2,8 +2,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ContractVerifierConfig { @@ -16,10 +14,6 @@ pub struct ContractVerifierConfig { } impl ContractVerifierConfig { - pub fn from_env() -> anyhow::Result { - envy_load("contract_verifier", "CONTRACT_VERIFIER_") - } - pub fn compilation_timeout(&self) -> Duration { Duration::from_secs(self.compilation_timeout) } @@ -28,33 +22,3 @@ impl ContractVerifierConfig { Duration::from_millis(self.polling_interval.unwrap_or(1000)) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ContractVerifierConfig { - ContractVerifierConfig { - compilation_timeout: 30, - polling_interval: Some(1000), - prometheus_port: 3314, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30 - CONTRACT_VERIFIER_POLLING_INTERVAL=1000 - CONTRACT_VERIFIER_PROMETHEUS_PORT=3314 - "#; - lock.set_env(config); - - let actual = ContractVerifierConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/contracts.rs b/core/lib/config/src/configs/contracts.rs index 5980a165cfed..a05027e0bf0d 100644 --- a/core/lib/config/src/configs/contracts.rs +++ b/core/lib/config/src/configs/contracts.rs @@ -2,9 +2,6 @@ use serde::Deserialize; // Workspace uses use zksync_basic_types::{Address, H256}; -// Local uses -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ProverAtGenesis { @@ -46,105 +43,40 @@ pub struct ContractsConfig { } impl ContractsConfig { - pub fn from_env() -> anyhow::Result { - envy_load("contracts", "CONTRACTS_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{addr, hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ContractsConfig { - ContractsConfig { - governance_addr: addr("d8dA6BF26964aF9D7eEd9e03E53415D37aA96045"), - mailbox_facet_addr: addr("0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd"), - executor_facet_addr: addr("18B631537801963A964211C0E86645c1aBfbB2d3"), - admin_facet_addr: addr("1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32"), - getters_facet_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - verifier_addr: addr("34782eE00206EAB6478F2692caa800e4A581687b"), - diamond_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), - diamond_upgrade_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), - diamond_proxy_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), - validator_timelock_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), - genesis_tx_hash: hash( - "b99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e", - ), - l1_erc20_bridge_proxy_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_erc20_bridge_impl_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l2_erc20_bridge_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_allow_list_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), - l1_weth_bridge_proxy_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), - l2_weth_bridge_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), - l2_testnet_paymaster_addr: Some(addr("FC073319977e314F251EAE6ae6bE76B0B3BAeeCF")), - recursion_scheduler_level_vk_hash: hash( - "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", - ), - recursion_node_level_vk_hash: hash( - "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", - ), - recursion_leaf_level_vk_hash: hash( - "0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210", - ), - recursion_circuits_set_vks_hash: hash( - "0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0", - ), - l1_multicall3_addr: addr("0xcA11bde05977b3631167028862bE2a173976CA11"), - fri_recursion_scheduler_level_vk_hash: hash( - "0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d", - ), - fri_recursion_node_level_vk_hash: hash( - "0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080", - ), - fri_recursion_leaf_level_vk_hash: hash( - "0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828", - ), + /// Creates a mock instance of `ContractsConfig` to be used in tests. + /// No data in the created object is valid. + /// Every contract address is set to a random but unique non-zero value. + /// Same goes for hashes. + pub fn for_tests() -> Self { + Self { + mailbox_facet_addr: Address::repeat_byte(0x01), + executor_facet_addr: Address::repeat_byte(0x02), + admin_facet_addr: Address::repeat_byte(0x03), + getters_facet_addr: Address::repeat_byte(0x05), + verifier_addr: Address::repeat_byte(0x06), + diamond_init_addr: Address::repeat_byte(0x07), + diamond_upgrade_init_addr: Address::repeat_byte(0x08), + diamond_proxy_addr: Address::repeat_byte(0x09), + validator_timelock_addr: Address::repeat_byte(0x0a), + genesis_tx_hash: H256::repeat_byte(0x01), + l1_erc20_bridge_proxy_addr: Address::repeat_byte(0x0b), + l1_erc20_bridge_impl_addr: Address::repeat_byte(0x0c), + l2_erc20_bridge_addr: Address::repeat_byte(0x0d), + l1_weth_bridge_proxy_addr: Some(Address::repeat_byte(0x0e)), + l2_weth_bridge_addr: Some(Address::repeat_byte(0x0f)), + l1_allow_list_addr: Address::repeat_byte(0x10), + l2_testnet_paymaster_addr: Some(Address::repeat_byte(0x11)), + recursion_scheduler_level_vk_hash: H256::repeat_byte(0x02), + recursion_node_level_vk_hash: H256::repeat_byte(0x03), + recursion_leaf_level_vk_hash: H256::repeat_byte(0x04), + recursion_circuits_set_vks_hash: H256::repeat_byte(0x05), + l1_multicall3_addr: Address::repeat_byte(0x12), + fri_recursion_scheduler_level_vk_hash: H256::repeat_byte(0x06), + fri_recursion_node_level_vk_hash: H256::repeat_byte(0x07), + fri_recursion_leaf_level_vk_hash: H256::repeat_byte(0x08), + governance_addr: Address::repeat_byte(0x13), prover_at_genesis: ProverAtGenesis::Fri, - snark_wrapper_vk_hash: hash( - "0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2", - ), + snark_wrapper_vk_hash: H256::repeat_byte(0x09), } } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" -CONTRACTS_GOVERNANCE_ADDR="0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" -CONTRACTS_MAILBOX_FACET_ADDR="0x0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd" -CONTRACTS_EXECUTOR_FACET_ADDR="0x18B631537801963A964211C0E86645c1aBfbB2d3" -CONTRACTS_ADMIN_FACET_ADDR="0x1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32" -CONTRACTS_GETTERS_FACET_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_VERIFIER_ADDR="0x34782eE00206EAB6478F2692caa800e4A581687b" -CONTRACTS_DIAMOND_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" -CONTRACTS_DIAMOND_UPGRADE_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" -CONTRACTS_DIAMOND_PROXY_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" -CONTRACTS_VALIDATOR_TIMELOCK_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" -CONTRACTS_GENESIS_TX_HASH="0xb99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e" -CONTRACTS_L1_ERC20_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_ALLOW_LIST_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_ERC20_BRIDGE_IMPL_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_ERC20_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L1_WETH_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_WETH_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" -CONTRACTS_L2_TESTNET_PAYMASTER_ADDR="FC073319977e314F251EAE6ae6bE76B0B3BAeeCF" -CONTRACTS_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" -CONTRACTS_RECURSION_NODE_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" -CONTRACTS_RECURSION_LEAF_LEVEL_VK_HASH="0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210" -CONTRACTS_RECURSION_CIRCUITS_SET_VKS_HASH="0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0" -CONTRACTS_L1_MULTICALL3_ADDR="0xcA11bde05977b3631167028862bE2a173976CA11" -CONTRACTS_FRI_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d" -CONTRACTS_FRI_RECURSION_NODE_LEVEL_VK_HASH="0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080" -CONTRACTS_FRI_RECURSION_LEAF_LEVEL_VK_HASH="0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828" -CONTRACTS_PROVER_AT_GENESIS="fri" -CONTRACTS_SNARK_WRAPPER_VK_HASH="0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2" - "#; - lock.set_env(config); - - let actual = ContractsConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } } diff --git a/core/lib/config/src/configs/database.rs b/core/lib/config/src/configs/database.rs index 0628d8d45653..e694401aa939 100644 --- a/core/lib/config/src/configs/database.rs +++ b/core/lib/config/src/configs/database.rs @@ -2,8 +2,6 @@ use serde::{Deserialize, Serialize}; use std::time::Duration; -use super::envy_load; - /// Mode of operation for the Merkle tree. /// /// The mode does not influence how tree data is stored; i.e., a mode can be switched on the fly. @@ -145,13 +143,6 @@ impl DBConfig { 60_000 } - pub fn from_env() -> anyhow::Result { - Ok(Self { - merkle_tree: envy_load("database_merkle_tree", "DATABASE_MERKLE_TREE_")?, - ..envy_load("database", "DATABASE_")? - }) - } - /// Returns the Postgres statement timeout. pub fn statement_timeout(&self) -> Option { self.statement_timeout_sec.map(Duration::from_secs) @@ -161,89 +152,3 @@ impl DBConfig { Duration::from_millis(self.backup_interval_ms) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - DATABASE_STATE_KEEPER_DB_PATH="/db/state_keeper" - DATABASE_MERKLE_TREE_BACKUP_PATH="/db/backups" - DATABASE_MERKLE_TREE_PATH="/db/tree" - DATABASE_MERKLE_TREE_MODE=lightweight - DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=250 - DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB=512 - DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC=60 - DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50 - DATABASE_BACKUP_COUNT=5 - DATABASE_BACKUP_INTERVAL_MS=60000 - "#; - lock.set_env(config); - - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.state_keeper_db_path, "/db/state_keeper"); - assert_eq!(db_config.merkle_tree.path, "/db/tree"); - assert_eq!(db_config.merkle_tree.backup_path, "/db/backups"); - assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Lightweight); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 250); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); - assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 512); - assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 60); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); - } - - #[test] - fn from_empty_env() { - let mut lock = MUTEX.lock(); - lock.remove_env(&[ - "DATABASE_STATE_KEEPER_DB_PATH", - "DATABASE_MERKLE_TREE_BACKUP_PATH", - "DATABASE_MERKLE_TREE_PATH", - "DATABASE_MERKLE_TREE_MODE", - "DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE", - "DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB", - "DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB", - "DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC", - "DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER", - "DATABASE_BACKUP_COUNT", - "DATABASE_BACKUP_INTERVAL_MS", - ]); - - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.state_keeper_db_path, "./db/state_keeper"); - assert_eq!(db_config.merkle_tree.path, "./db/lightweight-new"); - assert_eq!(db_config.merkle_tree.backup_path, "./db/backups"); - assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Full); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 500); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 20); - assert_eq!(db_config.merkle_tree.block_cache_size_mb, 128); - assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 256); - assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 30); - assert_eq!(db_config.backup_count, 5); - assert_eq!(db_config.backup_interval().as_secs(), 60); - - // Check that new env variable for Merkle tree path is supported - lock.set_env("DATABASE_MERKLE_TREE_PATH=/db/tree/main"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.path, "/db/tree/main"); - - lock.set_env("DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=200"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 200); - - lock.set_env("DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB=256"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.block_cache_size_mb, 256); - - lock.set_env("DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50"); - let db_config = DBConfig::from_env().unwrap(); - assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); - } -} diff --git a/core/lib/config/src/configs/eth_client.rs b/core/lib/config/src/configs/eth_client.rs index 4cebdacc7fcd..69b722f1f704 100644 --- a/core/lib/config/src/configs/eth_client.rs +++ b/core/lib/config/src/configs/eth_client.rs @@ -1,7 +1,5 @@ // External uses use serde::Deserialize; -// Local uses -use super::envy_load; /// Configuration for the Ethereum gateways. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -11,44 +9,3 @@ pub struct ETHClientConfig { /// Address of the Ethereum node API. pub web3_url: String, } - -impl ETHClientConfig { - pub fn from_env() -> anyhow::Result { - let config: Self = envy_load("eth_client", "ETH_CLIENT_")?; - if config.web3_url.find(',').is_some() { - anyhow::bail!( - "Multiple web3 URLs aren't supported anymore. Provided invalid value: {}", - config.web3_url - ); - } - Ok(config) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHClientConfig { - ETHClientConfig { - chain_id: 9, - web3_url: "http://127.0.0.1:8545".into(), - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_CLIENT_CHAIN_ID="9" - ETH_CLIENT_WEB3_URL="http://127.0.0.1:8545" - "#; - lock.set_env(config); - - let actual = ETHClientConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/eth_sender.rs b/core/lib/config/src/configs/eth_sender.rs index 5516340035f9..3d036483347f 100644 --- a/core/lib/config/src/configs/eth_sender.rs +++ b/core/lib/config/src/configs/eth_sender.rs @@ -1,12 +1,9 @@ // Built-in uses use std::time::Duration; // External uses -use anyhow::Context as _; use serde::Deserialize; // Workspace uses use zksync_basic_types::H256; -// Local uses -use super::envy_load; /// Configuration for the Ethereum sender crate. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -18,11 +15,40 @@ pub struct ETHSenderConfig { } impl ETHSenderConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - sender: SenderConfig::from_env().context("SenderConfig")?, - gas_adjuster: GasAdjusterConfig::from_env().context("GasAdjusterConfig")?, - }) + /// Creates a mock configuration object suitable for unit tests. + /// Values inside match the config used for localhost development. + pub fn for_tests() -> Self { + Self { + sender: SenderConfig { + aggregated_proof_sizes: vec![1, 4], + wait_confirmations: Some(1), + tx_poll_period: 1, + aggregate_tx_poll_period: 1, + max_txs_in_flight: 30, + proof_sending_mode: ProofSendingMode::SkipEveryProof, + max_aggregated_tx_gas: 4000000, + max_eth_tx_data_size: 6000000, + max_aggregated_blocks_to_commit: 10, + max_aggregated_blocks_to_execute: 10, + aggregated_block_commit_deadline: 1, + aggregated_block_prove_deadline: 10, + aggregated_block_execute_deadline: 10, + timestamp_criteria_max_allowed_lag: 30, + l1_batch_min_age_before_execute_seconds: None, + max_acceptable_priority_fee_in_gwei: 100000000000, + proof_loading_mode: ProofLoadingMode::OldProofFromDb, + }, + gas_adjuster: GasAdjusterConfig { + default_priority_fee_per_gas: 1000000000, + max_base_fee_samples: 10000, + pricing_formula_parameter_a: 1.5, + pricing_formula_parameter_b: 1.0005, + internal_l1_pricing_multiplier: 0.8, + internal_enforced_l1_gas_price: None, + poll_period: 5, + max_l1_gas_price: None, + }, + } } } @@ -79,6 +105,7 @@ impl SenderConfig { pub fn tx_poll_period(&self) -> Duration { Duration::from_secs(self.tx_poll_period) } + /// Converts `self.aggregate_tx_poll_period` into `Duration`. pub fn aggregate_tx_poll_period(&self) -> Duration { Duration::from_secs(self.aggregate_tx_poll_period) @@ -90,10 +117,6 @@ impl SenderConfig { .ok() .map(|pk| pk.parse().unwrap()) } - - pub fn from_env() -> anyhow::Result { - envy_load("eth_sender", "ETH_SENDER_SENDER_") - } } #[derive(Debug, Deserialize, Copy, Clone, PartialEq)] @@ -125,103 +148,4 @@ impl GasAdjusterConfig { pub fn max_l1_gas_price(&self) -> u64 { self.max_l1_gas_price.unwrap_or(u64::MAX) } - - pub fn from_env() -> anyhow::Result { - envy_load("eth_sender.gas_adjuster", "ETH_SENDER_GAS_ADJUSTER_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::{hash, EnvMutex}; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHSenderConfig { - ETHSenderConfig { - sender: SenderConfig { - aggregated_proof_sizes: vec![1, 5], - aggregated_block_commit_deadline: 30, - aggregated_block_prove_deadline: 3_000, - aggregated_block_execute_deadline: 4_000, - max_aggregated_tx_gas: 4_000_000, - max_eth_tx_data_size: 120_000, - - timestamp_criteria_max_allowed_lag: 30, - max_aggregated_blocks_to_commit: 3, - max_aggregated_blocks_to_execute: 4, - wait_confirmations: Some(1), - tx_poll_period: 3, - aggregate_tx_poll_period: 3, - max_txs_in_flight: 3, - proof_sending_mode: ProofSendingMode::SkipEveryProof, - l1_batch_min_age_before_execute_seconds: Some(1000), - max_acceptable_priority_fee_in_gwei: 100_000_000_000, - proof_loading_mode: ProofLoadingMode::OldProofFromDb, - }, - gas_adjuster: GasAdjusterConfig { - default_priority_fee_per_gas: 20000000000, - max_base_fee_samples: 10000, - pricing_formula_parameter_a: 1.5, - pricing_formula_parameter_b: 1.0005, - internal_l1_pricing_multiplier: 0.8, - internal_enforced_l1_gas_price: None, - poll_period: 15, - max_l1_gas_price: Some(100000000), - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_SENDER_SENDER_WAIT_CONFIRMATIONS="1" - ETH_SENDER_SENDER_TX_POLL_PERIOD="3" - ETH_SENDER_SENDER_AGGREGATE_TX_POLL_PERIOD="3" - ETH_SENDER_SENDER_MAX_TXS_IN_FLIGHT="3" - ETH_SENDER_SENDER_OPERATOR_PRIVATE_KEY="0x27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be" - ETH_SENDER_SENDER_PROOF_SENDING_MODE="SkipEveryProof" - ETH_SENDER_GAS_ADJUSTER_DEFAULT_PRIORITY_FEE_PER_GAS="20000000000" - ETH_SENDER_GAS_ADJUSTER_MAX_BASE_FEE_SAMPLES="10000" - ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_A="1.5" - ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_B="1.0005" - ETH_SENDER_GAS_ADJUSTER_INTERNAL_L1_PRICING_MULTIPLIER="0.8" - ETH_SENDER_GAS_ADJUSTER_POLL_PERIOD="15" - ETH_SENDER_GAS_ADJUSTER_MAX_L1_GAS_PRICE="100000000" - ETH_SENDER_WAIT_FOR_PROOFS="false" - ETH_SENDER_SENDER_AGGREGATED_PROOF_SIZES="1,5" - ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_COMMIT="3" - ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_EXECUTE="4" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_COMMIT_DEADLINE="30" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_PROVE_DEADLINE="3000" - ETH_SENDER_SENDER_AGGREGATED_BLOCK_EXECUTE_DEADLINE="4000" - ETH_SENDER_SENDER_TIMESTAMP_CRITERIA_MAX_ALLOWED_LAG="30" - ETH_SENDER_SENDER_MAX_AGGREGATED_TX_GAS="4000000" - ETH_SENDER_SENDER_MAX_ETH_TX_DATA_SIZE="120000" - ETH_SENDER_SENDER_L1_BATCH_MIN_AGE_BEFORE_EXECUTE_SECONDS="1000" - ETH_SENDER_SENDER_MAX_ACCEPTABLE_PRIORITY_FEE_IN_GWEI="100000000000" - ETH_SENDER_SENDER_PROOF_LOADING_MODE="OldProofFromDb" - "#; - lock.set_env(config); - - let actual = ETHSenderConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - assert_eq!( - actual.sender.private_key().unwrap(), - hash("27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be") - ); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - - assert_eq!( - config.sender.tx_poll_period(), - Duration::from_secs(config.sender.tx_poll_period) - ); - } } diff --git a/core/lib/config/src/configs/eth_watch.rs b/core/lib/config/src/configs/eth_watch.rs index a8662ddecb95..93d73ddf6bfd 100644 --- a/core/lib/config/src/configs/eth_watch.rs +++ b/core/lib/config/src/configs/eth_watch.rs @@ -2,8 +2,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; /// Configuration for the Ethereum sender crate. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -17,50 +15,8 @@ pub struct ETHWatchConfig { } impl ETHWatchConfig { - pub fn from_env() -> anyhow::Result { - envy_load("eth_watch", "ETH_WATCH_") - } - /// Converts `self.eth_node_poll_interval` into `Duration`. pub fn poll_interval(&self) -> Duration { Duration::from_millis(self.eth_node_poll_interval) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ETHWatchConfig { - ETHWatchConfig { - confirmations_for_eth_event: Some(0), - eth_node_poll_interval: 300, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - ETH_WATCH_CONFIRMATIONS_FOR_ETH_EVENT="0" - ETH_WATCH_ETH_NODE_POLL_INTERVAL="300" - "#; - lock.set_env(config); - - let actual = ETHWatchConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - /// Checks the correctness of the config helper methods. - #[test] - fn methods() { - let config = expected_config(); - assert_eq!( - config.poll_interval(), - Duration::from_millis(config.eth_node_poll_interval) - ); - } -} diff --git a/core/lib/config/src/configs/fetcher.rs b/core/lib/config/src/configs/fetcher.rs index ac3cff36b40b..b1a5fca4b24a 100644 --- a/core/lib/config/src/configs/fetcher.rs +++ b/core/lib/config/src/configs/fetcher.rs @@ -1,11 +1,5 @@ -use std::time::Duration; - -// Built-in uses -// External uses use serde::Deserialize; -// Workspace uses -// Local uses -use super::envy_load; +use std::time::Duration; #[derive(Debug, Deserialize, Clone, Copy, PartialEq)] pub enum TokenListSource { @@ -49,64 +43,3 @@ pub struct FetcherConfig { pub token_price: SingleFetcherConfig, pub token_trading_volume: SingleFetcherConfig, } - -impl FetcherConfig { - pub fn from_env() -> anyhow::Result { - Ok(Self { - token_list: envy_load("token_list", "FETCHER_TOKEN_LIST_")?, - token_price: envy_load("token_price", "FETCHER_TOKEN_PRICE_")?, - token_trading_volume: envy_load( - "token_trading_volume", - "FETCHER_TOKEN_TRADING_VOLUME_", - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FetcherConfig { - FetcherConfig { - token_list: SingleFetcherConfig { - source: TokenListSource::OneInch, - url: "http://127.0.0.1:1020".into(), - fetching_interval: 10, - }, - token_price: SingleFetcherConfig { - source: TokenPriceSource::CoinGecko, - url: "http://127.0.0.1:9876".into(), - fetching_interval: 7, - }, - token_trading_volume: SingleFetcherConfig { - source: TokenTradingVolumeSource::Uniswap, - url: "http://127.0.0.1:9975/graphql".to_string(), - fetching_interval: 5, - }, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FETCHER_TOKEN_LIST_SOURCE="OneInch" - FETCHER_TOKEN_LIST_URL="http://127.0.0.1:1020" - FETCHER_TOKEN_LIST_FETCHING_INTERVAL="10" - FETCHER_TOKEN_PRICE_SOURCE="CoinGecko" - FETCHER_TOKEN_PRICE_URL="http://127.0.0.1:9876" - FETCHER_TOKEN_PRICE_FETCHING_INTERVAL="7" - FETCHER_TOKEN_TRADING_VOLUME_SOURCE="Uniswap" - FETCHER_TOKEN_TRADING_VOLUME_URL="http://127.0.0.1:9975/graphql" - FETCHER_TOKEN_TRADING_VOLUME_FETCHING_INTERVAL="5" - "#; - lock.set_env(config); - - let actual = FetcherConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_proof_compressor.rs b/core/lib/config/src/configs/fri_proof_compressor.rs index e544b4f52064..bbf58f2d1c69 100644 --- a/core/lib/config/src/configs/fri_proof_compressor.rs +++ b/core/lib/config/src/configs/fri_proof_compressor.rs @@ -1,8 +1,6 @@ use serde::Deserialize; use std::time::Duration; -use super::envy_load; - /// Configuration for the fri proof compressor #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriProofCompressorConfig { @@ -29,56 +27,7 @@ pub struct FriProofCompressorConfig { } impl FriProofCompressorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_proof_compressor", "FRI_PROOF_COMPRESSOR_") - } - pub fn generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use crate::configs::test_utils::EnvMutex; - - use super::*; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProofCompressorConfig { - FriProofCompressorConfig { - compression_mode: 1, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - generation_timeout_in_secs: 3000, - max_attempts: 5, - universal_setup_path: "keys/setup/setup_2^26.key".to_string(), - universal_setup_download_url: - "https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" - .to_string(), - verify_wrapper_proof: false, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_PROOF_COMPRESSOR_COMPRESSION_MODE=1 - FRI_PROOF_COMPRESSOR_PROMETHEUS_LISTENER_PORT=3316 - FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSH_INTERVAL_MS=100 - FRI_PROOF_COMPRESSOR_GENERATION_TIMEOUT_IN_SECS=3000 - FRI_PROOF_COMPRESSOR_MAX_ATTEMPTS=5 - FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_PATH="keys/setup/setup_2^26.key" - FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_DOWNLOAD_URL="https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" - FRI_PROOF_COMPRESSOR_VERIFY_WRAPPER_PROOF=false - "#; - lock.set_env(config); - - let actual = FriProofCompressorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover.rs b/core/lib/config/src/configs/fri_prover.rs index 205f13e114c8..aab358a4adaf 100644 --- a/core/lib/config/src/configs/fri_prover.rs +++ b/core/lib/config/src/configs/fri_prover.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -28,59 +27,7 @@ pub struct FriProverConfig { } impl FriProverConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_prover", "FRI_PROVER_") - } - pub fn proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProverConfig { - FriProverConfig { - setup_data_path: "/usr/src/setup-data".to_string(), - prometheus_port: 3315, - max_attempts: 10, - generation_timeout_in_secs: 300, - base_layer_circuit_ids_to_be_verified: vec![1, 5], - recursive_layer_circuit_ids_to_be_verified: vec![1, 2, 3], - setup_load_mode: SetupLoadMode::FromDisk, - specialized_group_id: 10, - witness_vector_generator_thread_count: Some(5), - queue_capacity: 10, - witness_vector_receiver_port: 3316, - shall_save_to_public_bucket: true, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_PROVER_SETUP_DATA_PATH="/usr/src/setup-data" - FRI_PROVER_PROMETHEUS_PORT="3315" - FRI_PROVER_MAX_ATTEMPTS="10" - FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300" - FRI_PROVER_BASE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,5" - FRI_PROVER_RECURSIVE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,2,3" - FRI_PROVER_SETUP_LOAD_MODE="FromDisk" - FRI_PROVER_SPECIALIZED_GROUP_ID="10" - FRI_PROVER_WITNESS_VECTOR_GENERATOR_THREAD_COUNT="5" - FRI_PROVER_QUEUE_CAPACITY="10" - FRI_PROVER_WITNESS_VECTOR_RECEIVER_PORT="3316" - FRI_PROVER_SHALL_SAVE_TO_PUBLIC_BUCKET=true - "#; - lock.set_env(config); - - let actual = FriProverConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover_gateway.rs b/core/lib/config/src/configs/fri_prover_gateway.rs index f8eb7fbd2ca7..652c7d1bc0f5 100644 --- a/core/lib/config/src/configs/fri_prover_gateway.rs +++ b/core/lib/config/src/configs/fri_prover_gateway.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -14,44 +13,7 @@ pub struct FriProverGatewayConfig { } impl FriProverGatewayConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_prover_gateway", "FRI_PROVER_GATEWAY_") - } - pub fn api_poll_duration(&self) -> Duration { Duration::from_secs(self.api_poll_duration_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriProverGatewayConfig { - FriProverGatewayConfig { - api_url: "http://private-dns-for-server".to_string(), - api_poll_duration_secs: 100, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - } - } - - #[test] - fn from_env() { - let config = r#" - FRI_PROVER_GATEWAY_API_URL="http://private-dns-for-server" - FRI_PROVER_GATEWAY_API_POLL_DURATION_SECS="100" - FRI_PROVER_GATEWAY_PROMETHEUS_LISTENER_PORT=3316 - FRI_PROVER_GATEWAY_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_PROVER_GATEWAY_PROMETHEUS_PUSH_INTERVAL_MS=100 - "#; - let mut lock = MUTEX.lock(); - lock.set_env(config); - let actual = FriProverGatewayConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_prover_group.rs b/core/lib/config/src/configs/fri_prover_group.rs index bce956cfffc0..966a4b324db6 100644 --- a/core/lib/config/src/configs/fri_prover_group.rs +++ b/core/lib/config/src/configs/fri_prover_group.rs @@ -1,51 +1,8 @@ use serde::Deserialize; -use std::collections::{HashMap, HashSet}; -use std::env; +use std::collections::HashSet; use zksync_basic_types::basic_fri_types::CircuitIdRoundTuple; -fn load_from_env_variable() -> HashMap> { - // Prepare a hash map to store the mapping of group to a vector of tuples - let mut groups: HashMap> = (0..=12) - .map(|i| (format!("group_{}", i), HashSet::new())) - .collect(); - - // Separate environment variables into Circuit Id and Aggregation Round - let mut circuit_ids = HashMap::new(); - let mut aggregation_rounds = HashMap::new(); - for (key, value) in env::vars() { - if key.contains("_CIRCUIT_ID") { - circuit_ids.insert(key, value); - } else if key.contains("_AGGREGATION_ROUND") { - aggregation_rounds.insert(key, value); - } - } - - // Iterate over all circuit id variables - for (key, value_str) in circuit_ids { - let key_parts: Vec<&str> = key.split('_').collect(); - if let (Some(group_key), Some(value), Some(index_str)) = ( - key_parts.get(4), - value_str.parse::().ok(), - key_parts.get(5), - ) { - let round_key = format!( - "FRI_PROVER_GROUP_GROUP_{}_{}_AGGREGATION_ROUND", - group_key, index_str - ); - if let Some(round_str) = aggregation_rounds.get(&round_key) { - if let Ok(round) = round_str.parse::() { - let tuple = CircuitIdRoundTuple::new(value, round); - if let Some(group) = groups.get_mut(&format!("group_{}", group_key)) { - group.insert(tuple); - } - } - } - } - } - groups -} - /// Configuration for the grouping of specialized provers. #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriProverGroupConfig { @@ -63,29 +20,7 @@ pub struct FriProverGroupConfig { pub group_11: HashSet, pub group_12: HashSet, } - impl FriProverGroupConfig { - pub fn from_env() -> Self { - let mut groups = load_from_env_variable(); - let config = FriProverGroupConfig { - group_0: groups.remove("group_0").unwrap_or_default(), - group_1: groups.remove("group_1").unwrap_or_default(), - group_2: groups.remove("group_2").unwrap_or_default(), - group_3: groups.remove("group_3").unwrap_or_default(), - group_4: groups.remove("group_4").unwrap_or_default(), - group_5: groups.remove("group_5").unwrap_or_default(), - group_6: groups.remove("group_6").unwrap_or_default(), - group_7: groups.remove("group_7").unwrap_or_default(), - group_8: groups.remove("group_8").unwrap_or_default(), - group_9: groups.remove("group_9").unwrap_or_default(), - group_10: groups.remove("group_10").unwrap_or_default(), - group_11: groups.remove("group_11").unwrap_or_default(), - group_12: groups.remove("group_12").unwrap_or_default(), - }; - config.validate(); - config - } - pub fn get_circuit_ids_for_group_id(&self, group_id: u8) -> Option> { match group_id { 0 => Some(self.group_0.clone().into_iter().collect()), @@ -117,7 +52,7 @@ impl FriProverGroupConfig { /// In aggregation round 1, the circuit ids should be 3 to 15. /// In aggregation round 2, the circuit ids should be 2. /// In aggregation round 3, the circuit ids should be 1. - fn validate(&self) { + pub fn validate(&self) { let mut rounds: Vec> = vec![Vec::new(); 4]; let groups = [ &self.group_0, @@ -257,174 +192,3 @@ impl FriProverGroupConfig { } } } - -#[cfg(test)] -mod tests { - use super::*; - - fn expected_config() -> FriProverGroupConfig { - FriProverGroupConfig { - group_0: vec![ - CircuitIdRoundTuple::new(1, 3), - CircuitIdRoundTuple::new(2, 2), - ] - .into_iter() - .collect::>(), - group_1: vec![CircuitIdRoundTuple::new(1, 0)] - .into_iter() - .collect::>(), - group_2: vec![ - CircuitIdRoundTuple::new(2, 0), - CircuitIdRoundTuple::new(4, 0), - CircuitIdRoundTuple::new(6, 0), - CircuitIdRoundTuple::new(9, 0), - ] - .into_iter() - .collect::>(), - group_3: vec![CircuitIdRoundTuple::new(3, 0)] - .into_iter() - .collect::>(), - group_4: vec![ - CircuitIdRoundTuple::new(11, 0), - CircuitIdRoundTuple::new(12, 0), - CircuitIdRoundTuple::new(13, 0), - ] - .into_iter() - .collect::>(), - group_5: vec![CircuitIdRoundTuple::new(5, 0)] - .into_iter() - .collect::>(), - group_6: vec![CircuitIdRoundTuple::new(3, 1)] - .into_iter() - .collect::>(), - group_7: vec![CircuitIdRoundTuple::new(7, 0)] - .into_iter() - .collect::>(), - group_8: vec![CircuitIdRoundTuple::new(8, 0)] - .into_iter() - .collect::>(), - group_9: vec![ - CircuitIdRoundTuple::new(12, 1), - CircuitIdRoundTuple::new(13, 1), - CircuitIdRoundTuple::new(14, 1), - CircuitIdRoundTuple::new(15, 1), - ] - .into_iter() - .collect::>(), - group_10: vec![CircuitIdRoundTuple::new(10, 0)] - .into_iter() - .collect::>(), - group_11: vec![ - CircuitIdRoundTuple::new(7, 1), - CircuitIdRoundTuple::new(8, 1), - CircuitIdRoundTuple::new(10, 1), - CircuitIdRoundTuple::new(11, 1), - ] - .into_iter() - .collect::>(), - group_12: vec![ - CircuitIdRoundTuple::new(4, 1), - CircuitIdRoundTuple::new(5, 1), - CircuitIdRoundTuple::new(6, 1), - CircuitIdRoundTuple::new(9, 1), - ] - .into_iter() - .collect::>(), - } - } - - #[test] - fn from_env() { - let groups = [ - ("FRI_PROVER_GROUP_GROUP_0_0", CircuitIdRoundTuple::new(1, 3)), - ("FRI_PROVER_GROUP_GROUP_0_1", CircuitIdRoundTuple::new(2, 2)), - ("FRI_PROVER_GROUP_GROUP_1_0", CircuitIdRoundTuple::new(1, 0)), - ("FRI_PROVER_GROUP_GROUP_2_0", CircuitIdRoundTuple::new(2, 0)), - ("FRI_PROVER_GROUP_GROUP_2_1", CircuitIdRoundTuple::new(4, 0)), - ("FRI_PROVER_GROUP_GROUP_2_2", CircuitIdRoundTuple::new(6, 0)), - ("FRI_PROVER_GROUP_GROUP_2_3", CircuitIdRoundTuple::new(9, 0)), - ("FRI_PROVER_GROUP_GROUP_3_0", CircuitIdRoundTuple::new(3, 0)), - ( - "FRI_PROVER_GROUP_GROUP_4_0", - CircuitIdRoundTuple::new(11, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_4_1", - CircuitIdRoundTuple::new(12, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_4_2", - CircuitIdRoundTuple::new(13, 0), - ), - ("FRI_PROVER_GROUP_GROUP_5_0", CircuitIdRoundTuple::new(5, 0)), - ("FRI_PROVER_GROUP_GROUP_6_0", CircuitIdRoundTuple::new(3, 1)), - ("FRI_PROVER_GROUP_GROUP_7_0", CircuitIdRoundTuple::new(7, 0)), - ("FRI_PROVER_GROUP_GROUP_8_0", CircuitIdRoundTuple::new(8, 0)), - ( - "FRI_PROVER_GROUP_GROUP_9_0", - CircuitIdRoundTuple::new(12, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_1", - CircuitIdRoundTuple::new(13, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_2", - CircuitIdRoundTuple::new(14, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_9_3", - CircuitIdRoundTuple::new(15, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_10_0", - CircuitIdRoundTuple::new(10, 0), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_0", - CircuitIdRoundTuple::new(7, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_1", - CircuitIdRoundTuple::new(8, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_2", - CircuitIdRoundTuple::new(10, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_11_3", - CircuitIdRoundTuple::new(11, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_0", - CircuitIdRoundTuple::new(4, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_1", - CircuitIdRoundTuple::new(5, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_2", - CircuitIdRoundTuple::new(6, 1), - ), - ( - "FRI_PROVER_GROUP_GROUP_12_3", - CircuitIdRoundTuple::new(9, 1), - ), - ]; - - for (key_base, circuit_round_tuple) in &groups { - let circuit_id_key = format!("{}_CIRCUIT_ID", key_base); - let aggregation_round_key = format!("{}_AGGREGATION_ROUND", key_base); - env::set_var(&circuit_id_key, circuit_round_tuple.circuit_id.to_string()); - env::set_var( - &aggregation_round_key, - circuit_round_tuple.aggregation_round.to_string(), - ); - } - - let actual = FriProverGroupConfig::from_env(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_witness_generator.rs b/core/lib/config/src/configs/fri_witness_generator.rs index 4abc839635af..17b6d773c6fe 100644 --- a/core/lib/config/src/configs/fri_witness_generator.rs +++ b/core/lib/config/src/configs/fri_witness_generator.rs @@ -4,9 +4,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; - /// Configuration for the fri witness generation #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriWitnessGeneratorConfig { @@ -28,12 +25,7 @@ pub struct FriWitnessGeneratorConfig { // whether to write to public GCS bucket for https://github.com/matter-labs/era-boojum-validator-cli pub shall_save_to_public_bucket: bool, } - impl FriWitnessGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("fri_witness", "FRI_WITNESS_") - } - pub fn witness_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -42,40 +34,3 @@ impl FriWitnessGeneratorConfig { self.last_l1_batch_to_process.unwrap_or(u32::MAX) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriWitnessGeneratorConfig { - FriWitnessGeneratorConfig { - generation_timeout_in_secs: 900u16, - max_attempts: 4, - blocks_proving_percentage: Some(30), - dump_arguments_for_blocks: vec![2, 3], - last_l1_batch_to_process: None, - force_process_block: Some(1), - shall_save_to_public_bucket: true, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_WITNESS_GENERATION_TIMEOUT_IN_SECS=900 - FRI_WITNESS_MAX_ATTEMPTS=4 - FRI_WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" - FRI_WITNESS_BLOCKS_PROVING_PERCENTAGE="30" - FRI_WITNESS_FORCE_PROCESS_BLOCK="1" - FRI_WITNESS_SHALL_SAVE_TO_PUBLIC_BUCKET=true - "#; - lock.set_env(config); - - let actual = FriWitnessGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/fri_witness_vector_generator.rs b/core/lib/config/src/configs/fri_witness_vector_generator.rs index 0bb2ddac318d..98ce7ea9390b 100644 --- a/core/lib/config/src/configs/fri_witness_vector_generator.rs +++ b/core/lib/config/src/configs/fri_witness_vector_generator.rs @@ -2,8 +2,6 @@ use std::time::Duration; use serde::Deserialize; -use super::envy_load; - /// Configuration for the witness vector generator #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct FriWitnessVectorGeneratorConfig { @@ -25,13 +23,6 @@ pub struct FriWitnessVectorGeneratorConfig { } impl FriWitnessVectorGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load( - "fri_witness_vector_generator", - "FRI_WITNESS_VECTOR_GENERATOR_", - ) - } - pub fn prover_instance_wait_timeout(&self) -> Duration { Duration::from_secs(self.prover_instance_wait_timeout_in_secs as u64) } @@ -44,41 +35,3 @@ impl FriWitnessVectorGeneratorConfig { Duration::from_secs(self.max_prover_reservation_duration_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> FriWitnessVectorGeneratorConfig { - FriWitnessVectorGeneratorConfig { - max_prover_reservation_duration_in_secs: 1000u16, - prover_instance_wait_timeout_in_secs: 1000u16, - prover_instance_poll_time_in_milli_secs: 250u16, - prometheus_listener_port: 3316, - prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), - prometheus_push_interval_ms: Some(100), - specialized_group_id: 1, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - FRI_WITNESS_VECTOR_GENERATOR_MAX_PROVER_RESERVATION_DURATION_IN_SECS=1000 - FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 - FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3316 - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" - FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSH_INTERVAL_MS=100 - FRI_WITNESS_VECTOR_GENERATOR_SPECIALIZED_GROUP_ID=1 - "#; - lock.set_env(config); - - let actual = FriWitnessVectorGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/house_keeper.rs b/core/lib/config/src/configs/house_keeper.rs index 0215f6676404..f7eb35ac3708 100644 --- a/core/lib/config/src/configs/house_keeper.rs +++ b/core/lib/config/src/configs/house_keeper.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - /// Configuration for the house keeper. #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct HouseKeeperConfig { @@ -20,61 +18,3 @@ pub struct HouseKeeperConfig { pub fri_proof_compressor_job_retrying_interval_ms: u64, pub fri_proof_compressor_stats_reporting_interval_ms: u64, } - -impl HouseKeeperConfig { - pub fn from_env() -> anyhow::Result { - envy_load("house_keeper", "HOUSE_KEEPER_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> HouseKeeperConfig { - HouseKeeperConfig { - l1_batch_metrics_reporting_interval_ms: 10_000, - blob_cleaning_interval_ms: 60_000, - gpu_prover_queue_reporting_interval_ms: 10_000, - prover_job_retrying_interval_ms: 300_000, - prover_stats_reporting_interval_ms: 5_000, - witness_job_moving_interval_ms: 30_000, - witness_generator_stats_reporting_interval_ms: 10_000, - fri_witness_job_moving_interval_ms: 40_000, - fri_prover_job_retrying_interval_ms: 30_000, - fri_witness_generator_job_retrying_interval_ms: 30_000, - prover_db_pool_size: 2, - fri_prover_stats_reporting_interval_ms: 30_000, - fri_proof_compressor_job_retrying_interval_ms: 30_000, - fri_proof_compressor_stats_reporting_interval_ms: 30_000, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - HOUSE_KEEPER_L1_BATCH_METRICS_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_BLOB_CLEANING_INTERVAL_MS="60000" - HOUSE_KEEPER_GPU_PROVER_QUEUE_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="300000" - HOUSE_KEEPER_PROVER_STATS_REPORTING_INTERVAL_MS="5000" - HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000" - HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000" - HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000" - HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" - HOUSE_KEEPER_PROVER_DB_POOL_SIZE="2" - HOUSE_KEEPER_FRI_PROVER_STATS_REPORTING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_STATS_REPORTING_INTERVAL_MS="30000" - HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_JOB_RETRYING_INTERVAL_MS="30000" - "#; - lock.set_env(config); - - let actual = HouseKeeperConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/mod.rs b/core/lib/config/src/configs/mod.rs index 4fac42c18ef0..f23b6d414e33 100644 --- a/core/lib/config/src/configs/mod.rs +++ b/core/lib/config/src/configs/mod.rs @@ -13,9 +13,6 @@ pub use self::{ witness_generator::WitnessGeneratorConfig, }; -use anyhow::Context as _; -use serde::de::DeserializeOwned; - pub mod alerts; pub mod api; pub mod chain; @@ -41,18 +38,4 @@ pub mod prover_group; pub mod utils; pub mod witness_generator; -#[cfg(test)] -pub(crate) mod test_utils; - const BYTES_IN_MEGABYTE: usize = 1_024 * 1_024; - -/// Convenience function that loads the structure from the environment variable given the prefix. -/// Panics if the config cannot be loaded from the environment variables. -pub fn envy_load(name: &str, prefix: &str) -> anyhow::Result { - envy_try_load(prefix).with_context(|| format!("Cannot load config <{name}>")) -} - -/// Convenience function that loads the structure from the environment variable given the prefix. -pub fn envy_try_load(prefix: &str) -> Result { - envy::prefixed(prefix).from_env() -} diff --git a/core/lib/config/src/configs/object_store.rs b/core/lib/config/src/configs/object_store.rs index 379bb7d24f53..5524b6ade253 100644 --- a/core/lib/config/src/configs/object_store.rs +++ b/core/lib/config/src/configs/object_store.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - #[derive(Debug, Deserialize, Eq, PartialEq, Clone, Copy)] pub enum ObjectStoreMode { GCS, @@ -18,80 +16,3 @@ pub struct ObjectStoreConfig { pub gcs_credential_file_path: String, pub max_retries: u16, } - -impl ObjectStoreConfig { - pub fn from_env() -> anyhow::Result { - envy_load("object_store", "OBJECT_STORE_") - } - - pub fn public_from_env() -> anyhow::Result { - envy_load("public_object_store", "PUBLIC_OBJECT_STORE_") - } - - pub fn prover_from_env() -> anyhow::Result { - envy_load("prover_object_store", "PROVER_OBJECT_STORE_") - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config(bucket_base_url: &str) -> ObjectStoreConfig { - ObjectStoreConfig { - bucket_base_url: bucket_base_url.to_string(), - mode: ObjectStoreMode::FileBacked, - file_backed_base_path: "artifacts".to_string(), - gcs_credential_file_path: "/path/to/credentials.json".to_string(), - max_retries: 5, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - OBJECT_STORE_BUCKET_BASE_URL="/base/url" - OBJECT_STORE_MODE="FileBacked" - OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::from_env().unwrap(); - assert_eq!(actual, expected_config("/base/url")); - } - - #[test] - fn public_bucket_config_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - PUBLIC_OBJECT_STORE_BUCKET_BASE_URL="/public_base_url" - PUBLIC_OBJECT_STORE_MODE="FileBacked" - PUBLIC_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - PUBLIC_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - PUBLIC_OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::public_from_env().unwrap(); - assert_eq!(actual, expected_config("/public_base_url")); - } - - #[test] - fn prover_bucket_config_from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - PROVER_OBJECT_STORE_BUCKET_BASE_URL="/prover_base_url" - PROVER_OBJECT_STORE_MODE="FileBacked" - PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" - PROVER_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" - PROVER_OBJECT_STORE_MAX_RETRIES="5" - "#; - lock.set_env(config); - let actual = ObjectStoreConfig::prover_from_env().unwrap(); - assert_eq!(actual, expected_config("/prover_base_url")); - } -} diff --git a/core/lib/config/src/configs/proof_data_handler.rs b/core/lib/config/src/configs/proof_data_handler.rs index 577a01c055f5..e3efd6b7a4d7 100644 --- a/core/lib/config/src/configs/proof_data_handler.rs +++ b/core/lib/config/src/configs/proof_data_handler.rs @@ -1,4 +1,3 @@ -use super::envy_load; use serde::Deserialize; use std::time::Duration; @@ -15,44 +14,8 @@ pub struct ProofDataHandlerConfig { pub protocol_version_loading_mode: ProtocolVersionLoadingMode, pub fri_protocol_version_id: u16, } - impl ProofDataHandlerConfig { - pub fn from_env() -> anyhow::Result { - envy_load("proof_data_handler", "PROOF_DATA_HANDLER_") - } - pub fn proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.proof_generation_timeout_in_secs as u64) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProofDataHandlerConfig { - ProofDataHandlerConfig { - http_port: 3320, - proof_generation_timeout_in_secs: 18000, - protocol_version_loading_mode: ProtocolVersionLoadingMode::FromEnvVar, - fri_protocol_version_id: 2, - } - } - - #[test] - fn from_env() { - let config = r#" - PROOF_DATA_HANDLER_PROOF_GENERATION_TIMEOUT_IN_SECS="18000" - PROOF_DATA_HANDLER_HTTP_PORT="3320" - PROOF_DATA_HANDLER_PROTOCOL_VERSION_LOADING_MODE="FromEnvVar" - PROOF_DATA_HANDLER_FRI_PROTOCOL_VERSION_ID="2" - "#; - let mut lock = MUTEX.lock(); - lock.set_env(config); - let actual = ProofDataHandlerConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/prover.rs b/core/lib/config/src/configs/prover.rs index 483706b8244c..45ed7100f9f4 100644 --- a/core/lib/config/src/configs/prover.rs +++ b/core/lib/config/src/configs/prover.rs @@ -1,12 +1,7 @@ use std::time::Duration; -// Built-in uses -// External uses use serde::Deserialize; -// Local uses -use super::envy_load; - /// Configuration for the prover application #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct ProverConfig { @@ -64,195 +59,3 @@ impl ProverConfig { Duration::from_secs(self.generation_timeout_in_secs as u64) } } - -impl ProverConfigs { - pub fn from_env() -> anyhow::Result { - Ok(Self { - non_gpu: envy_load("non_gpu", "PROVER_NON_GPU_")?, - two_gpu_forty_gb_mem: envy_load( - "two_gpu_forty_gb_mem", - "PROVER_TWO_GPU_FORTY_GB_MEM_", - )?, - one_gpu_eighty_gb_mem: envy_load( - "one_gpu_eighty_gb_mem", - "PROVER_ONE_GPU_EIGHTY_GB_MEM_", - )?, - two_gpu_eighty_gb_mem: envy_load( - "two_gpu_eighty_gb_mem", - "PROVER_TWO_GPU_EIGHTY_GB_MEM_", - )?, - four_gpu_eighty_gb_mem: envy_load( - "four_gpu_eighty_gb_mem", - "PROVER_FOUR_GPU_EIGHTY_GB_MEM_", - )?, - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProverConfigs { - ProverConfigs { - non_gpu: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 2, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 0, - number_of_setup_slots: 2, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - two_gpu_forty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 2, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 1, - number_of_setup_slots: 5, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - one_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 4, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 2, - number_of_setup_slots: 5, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - two_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 9, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 3, - number_of_setup_slots: 9, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - four_gpu_eighty_gb_mem: ProverConfig { - prometheus_port: 3313, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - generation_timeout_in_secs: 2700u16, - number_of_threads: 18, - max_attempts: 4, - polling_duration_in_millis: 5, - setup_keys_path: "/usr/src/setup-keys".to_string(), - specialized_prover_group_id: 4, - number_of_setup_slots: 18, - assembly_receiver_port: 17791, - assembly_receiver_poll_time_in_millis: 250, - assembly_queue_capacity: 5, - }, - } - } - - const CONFIG: &str = r#" - PROVER_NON_GPU_PROMETHEUS_PORT="3313" - PROVER_NON_GPU_INITIAL_SETUP_KEY_PATH="key" - PROVER_NON_GPU_KEY_DOWNLOAD_URL="value" - PROVER_NON_GPU_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_NON_GPU_NUMBER_OF_THREADS="2" - PROVER_NON_GPU_MAX_ATTEMPTS="4" - PROVER_NON_GPU_POLLING_DURATION_IN_MILLIS=5 - PROVER_NON_GPU_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_NON_GPU_NUMBER_OF_SETUP_SLOTS=2 - PROVER_NON_GPU_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_NON_GPU_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_NON_GPU_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_NON_GPU_SPECIALIZED_PROVER_GROUP_ID=0 - - PROVER_TWO_GPU_FORTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_TWO_GPU_FORTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_TWO_GPU_FORTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_TWO_GPU_FORTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_THREADS="2" - PROVER_TWO_GPU_FORTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_TWO_GPU_FORTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_TWO_GPU_FORTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_TWO_GPU_FORTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=1 - - PROVER_ONE_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_ONE_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_ONE_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_ONE_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="4" - PROVER_ONE_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_ONE_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_ONE_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=2 - - PROVER_TWO_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_TWO_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_TWO_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_TWO_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="9" - PROVER_TWO_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_TWO_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_TWO_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=9 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_TWO_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=3 - - PROVER_FOUR_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="18" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" - PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=18 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 - PROVER_FOUR_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=4 - "#; - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - lock.set_env(CONFIG); - let actual = ProverConfigs::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/config/src/configs/prover_group.rs b/core/lib/config/src/configs/prover_group.rs index d4dfee82a203..2d40d47ba8c1 100644 --- a/core/lib/config/src/configs/prover_group.rs +++ b/core/lib/config/src/configs/prover_group.rs @@ -1,7 +1,5 @@ use serde::Deserialize; -use super::envy_load; - /// Configuration for the grouping of specialized provers. /// This config would be used by circuit-synthesizer and provers. #[derive(Debug, Deserialize, Clone, PartialEq)] @@ -26,10 +24,6 @@ pub struct ProverGroupConfig { } impl ProverGroupConfig { - pub fn from_env() -> anyhow::Result { - envy_load("prover_group", "PROVER_GROUP_") - } - pub fn get_circuit_ids_for_group_id(&self, group_id: u8) -> Option> { match group_id { 0 => Some(self.group_0_circuit_ids.clone()), @@ -70,143 +64,3 @@ impl ProverGroupConfig { .map(|(group_id, _)| group_id as u8) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> ProverGroupConfig { - ProverGroupConfig { - group_0_circuit_ids: vec![0, 18], - group_1_circuit_ids: vec![1, 4], - group_2_circuit_ids: vec![2, 5], - group_3_circuit_ids: vec![6, 7], - group_4_circuit_ids: vec![8, 9], - group_5_circuit_ids: vec![10, 11], - group_6_circuit_ids: vec![12, 13], - group_7_circuit_ids: vec![14, 15], - group_8_circuit_ids: vec![16, 17], - group_9_circuit_ids: vec![3], - region_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location".to_string(), - region_override: Some("us-central-1".to_string()), - zone_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/zone".to_string(), - zone_override: Some("us-central-1-b".to_string()), - synthesizer_per_gpu: 10, - } - } - - const CONFIG: &str = r#" - PROVER_GROUP_GROUP_0_CIRCUIT_IDS="0,18" - PROVER_GROUP_GROUP_1_CIRCUIT_IDS="1,4" - PROVER_GROUP_GROUP_2_CIRCUIT_IDS="2,5" - PROVER_GROUP_GROUP_3_CIRCUIT_IDS="6,7" - PROVER_GROUP_GROUP_4_CIRCUIT_IDS="8,9" - PROVER_GROUP_GROUP_5_CIRCUIT_IDS="10,11" - PROVER_GROUP_GROUP_6_CIRCUIT_IDS="12,13" - PROVER_GROUP_GROUP_7_CIRCUIT_IDS="14,15" - PROVER_GROUP_GROUP_8_CIRCUIT_IDS="16,17" - PROVER_GROUP_GROUP_9_CIRCUIT_IDS="3" - PROVER_GROUP_REGION_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location" - PROVER_GROUP_REGION_OVERRIDE="us-central-1" - PROVER_GROUP_ZONE_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/zone" - PROVER_GROUP_ZONE_OVERRIDE="us-central-1-b" - PROVER_GROUP_SYNTHESIZER_PER_GPU="10" - "#; - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - lock.set_env(CONFIG); - let actual = ProverGroupConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } - - #[test] - fn get_group_id_for_circuit_id() { - let prover_group_config = expected_config(); - - assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(0)); - assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(18)); - - assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(1)); - assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(4)); - - assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(2)); - assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(5)); - - assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(6)); - assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(7)); - - assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(8)); - assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(9)); - - assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(10)); - assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(11)); - - assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(12)); - assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(13)); - - assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(14)); - assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(15)); - - assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(16)); - assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(17)); - - assert_eq!(Some(9), prover_group_config.get_group_id_for_circuit_id(3)); - assert!(prover_group_config - .get_group_id_for_circuit_id(19) - .is_none()); - } - - #[test] - fn get_circuit_ids_for_group_id() { - let prover_group_config = expected_config(); - - assert_eq!( - Some(vec![0, 18]), - prover_group_config.get_circuit_ids_for_group_id(0) - ); - assert_eq!( - Some(vec![1, 4]), - prover_group_config.get_circuit_ids_for_group_id(1) - ); - assert_eq!( - Some(vec![2, 5]), - prover_group_config.get_circuit_ids_for_group_id(2) - ); - assert_eq!( - Some(vec![6, 7]), - prover_group_config.get_circuit_ids_for_group_id(3) - ); - assert_eq!( - Some(vec![8, 9]), - prover_group_config.get_circuit_ids_for_group_id(4) - ); - assert_eq!( - Some(vec![10, 11]), - prover_group_config.get_circuit_ids_for_group_id(5) - ); - assert_eq!( - Some(vec![12, 13]), - prover_group_config.get_circuit_ids_for_group_id(6) - ); - assert_eq!( - Some(vec![14, 15]), - prover_group_config.get_circuit_ids_for_group_id(7) - ); - assert_eq!( - Some(vec![16, 17]), - prover_group_config.get_circuit_ids_for_group_id(8) - ); - assert_eq!( - Some(vec![3]), - prover_group_config.get_circuit_ids_for_group_id(9) - ); - assert!(prover_group_config - .get_circuit_ids_for_group_id(10) - .is_none()); - } -} diff --git a/core/lib/config/src/configs/utils.rs b/core/lib/config/src/configs/utils.rs index 5de6abd97185..bfa9e7e7f3e6 100644 --- a/core/lib/config/src/configs/utils.rs +++ b/core/lib/config/src/configs/utils.rs @@ -2,8 +2,6 @@ use serde::Deserialize; use std::{env, time::Duration}; -use crate::configs::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub struct PrometheusConfig { /// Port to which the Prometheus exporter server is listening. @@ -15,10 +13,6 @@ pub struct PrometheusConfig { } impl PrometheusConfig { - pub fn from_env() -> anyhow::Result { - envy_load("prometheus", "API_PROMETHEUS_") - } - pub fn push_interval(&self) -> Duration { Duration::from_millis(self.push_interval_ms.unwrap_or(100)) } diff --git a/core/lib/config/src/configs/witness_generator.rs b/core/lib/config/src/configs/witness_generator.rs index de3bb29d08e9..010742e4b047 100644 --- a/core/lib/config/src/configs/witness_generator.rs +++ b/core/lib/config/src/configs/witness_generator.rs @@ -4,9 +4,6 @@ use std::time::Duration; // External uses use serde::Deserialize; -// Local uses -use super::envy_load; - #[derive(Debug, Deserialize, Clone, PartialEq)] pub enum BasicWitnessGeneratorDataSource { FromPostgres, @@ -38,10 +35,6 @@ pub struct WitnessGeneratorConfig { } impl WitnessGeneratorConfig { - pub fn from_env() -> anyhow::Result { - envy_load("witness", "WITNESS_") - } - pub fn witness_generation_timeout(&self) -> Duration { Duration::from_secs(self.generation_timeout_in_secs as u64) } @@ -50,42 +43,3 @@ impl WitnessGeneratorConfig { self.last_l1_batch_to_process.unwrap_or(u32::MAX) } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::configs::test_utils::EnvMutex; - - static MUTEX: EnvMutex = EnvMutex::new(); - - fn expected_config() -> WitnessGeneratorConfig { - WitnessGeneratorConfig { - generation_timeout_in_secs: 900_u16, - initial_setup_key_path: "key".to_owned(), - key_download_url: "value".to_owned(), - max_attempts: 4, - blocks_proving_percentage: Some(30), - dump_arguments_for_blocks: vec![2, 3], - last_l1_batch_to_process: None, - data_source: BasicWitnessGeneratorDataSource::FromBlob, - } - } - - #[test] - fn from_env() { - let mut lock = MUTEX.lock(); - let config = r#" - WITNESS_GENERATION_TIMEOUT_IN_SECS=900 - WITNESS_INITIAL_SETUP_KEY_PATH="key" - WITNESS_KEY_DOWNLOAD_URL="value" - WITNESS_MAX_ATTEMPTS=4 - WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" - WITNESS_BLOCKS_PROVING_PERCENTAGE="30" - WITNESS_DATA_SOURCE="FromBlob" - "#; - lock.set_env(config); - - let actual = WitnessGeneratorConfig::from_env().unwrap(); - assert_eq!(actual, expected_config()); - } -} diff --git a/core/lib/env_config/Cargo.toml b/core/lib/env_config/Cargo.toml new file mode 100644 index 000000000000..a084a3768bfd --- /dev/null +++ b/core/lib/env_config/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "zksync_env_config" +version = "0.1.0" +edition = "2018" +authors = ["The Matter Labs Team "] +homepage = "https://zksync.io/" +repository = "https://github.com/matter-labs/zksync-era" +license = "MIT OR Apache-2.0" +keywords = ["blockchain", "zksync"] +categories = ["cryptography"] + +[dependencies] +zksync_basic_types = { path = "../../lib/basic_types" } +zksync_config = { path = "../../lib/config" } + +anyhow = "1.0" +serde = "1.0" +envy = "0.4" diff --git a/core/lib/env_config/src/alerts.rs b/core/lib/env_config/src/alerts.rs new file mode 100644 index 000000000000..c72b23bbd9fa --- /dev/null +++ b/core/lib/env_config/src/alerts.rs @@ -0,0 +1,37 @@ +use crate::{envy_load, FromEnv}; +use zksync_config::configs::AlertsConfig; + +impl FromEnv for AlertsConfig { + fn from_env() -> anyhow::Result { + envy_load("sporadic_crypto_errors_substrs", "ALERTS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> AlertsConfig { + AlertsConfig { + sporadic_crypto_errors_substrs: vec![ + "EventDestroyErr".to_string(), + "Can't free memory of DeviceBuf".to_string(), + "value: PoisonError".to_string(), + ], + } + } + + #[test] + fn test_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ALERTS_SPORADIC_CRYPTO_ERRORS_SUBSTRS="EventDestroyErr,Can't free memory of DeviceBuf,value: PoisonError" + "#; + lock.set_env(config); + + assert_eq!(AlertsConfig::from_env().unwrap(), expected_config()); + } +} diff --git a/core/lib/env_config/src/api.rs b/core/lib/env_config/src/api.rs new file mode 100644 index 000000000000..92428fe45f1a --- /dev/null +++ b/core/lib/env_config/src/api.rs @@ -0,0 +1,151 @@ +use anyhow::Context as _; + +use crate::{envy_load, FromEnv}; +use zksync_config::configs::{ + api::{ + ContractVerificationApiConfig, HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig, + }, + ApiConfig, PrometheusConfig, +}; + +impl FromEnv for ApiConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + web3_json_rpc: Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig")?, + contract_verification: ContractVerificationApiConfig::from_env() + .context("ContractVerificationApiConfig")?, + prometheus: PrometheusConfig::from_env().context("PrometheusConfig")?, + healthcheck: HealthCheckConfig::from_env().context("HealthCheckConfig")?, + merkle_tree: MerkleTreeApiConfig::from_env().context("MerkleTreeApiConfig")?, + }) + } +} + +impl FromEnv for Web3JsonRpcConfig { + fn from_env() -> anyhow::Result { + envy_load("web3_json_rpc", "API_WEB3_JSON_RPC_") + } +} + +impl FromEnv for HealthCheckConfig { + fn from_env() -> anyhow::Result { + envy_load("healthcheck", "API_HEALTHCHECK_") + } +} + +impl FromEnv for ContractVerificationApiConfig { + fn from_env() -> anyhow::Result { + envy_load("contract_verification", "API_CONTRACT_VERIFICATION_") + } +} + +impl FromEnv for MerkleTreeApiConfig { + /// Loads configuration from env variables. + fn from_env() -> anyhow::Result { + envy_load("merkle_tree_api", "API_MERKLE_TREE_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{hash, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ApiConfig { + ApiConfig { + web3_json_rpc: Web3JsonRpcConfig { + http_port: 3050, + http_url: "http://127.0.0.1:3050".into(), + ws_port: 3051, + ws_url: "ws://127.0.0.1:3051".into(), + req_entities_limit: Some(10000), + filters_limit: Some(10000), + subscriptions_limit: Some(10000), + pubsub_polling_interval: Some(200), + threads_per_server: 128, + max_nonce_ahead: 5, + transactions_per_sec_limit: Some(1000), + request_timeout: Some(10), + account_pks: Some(vec![ + hash("0x0000000000000000000000000000000000000000000000000000000000000001"), + hash("0x0000000000000000000000000000000000000000000000000000000000000002"), + ]), + estimate_gas_scale_factor: 1.0f64, + gas_price_scale_factor: 1.2, + estimate_gas_acceptable_overestimation: 1000, + max_tx_size: 1000000, + vm_execution_cache_misses_limit: None, + vm_concurrency_limit: Some(512), + factory_deps_cache_size_mb: Some(128), + initial_writes_cache_size_mb: Some(32), + latest_values_cache_size_mb: Some(256), + http_threads: Some(128), + ws_threads: Some(256), + fee_history_limit: Some(100), + max_batch_request_size: Some(200), + max_response_body_size_mb: Some(10), + websocket_requests_per_minute_limit: Some(10), + }, + contract_verification: ContractVerificationApiConfig { + port: 3070, + url: "http://127.0.0.1:3070".into(), + threads_per_server: 128, + }, + prometheus: PrometheusConfig { + listener_port: 3312, + pushgateway_url: "http://127.0.0.1:9091".into(), + push_interval_ms: Some(100), + }, + healthcheck: HealthCheckConfig { port: 8081 }, + merkle_tree: MerkleTreeApiConfig { port: 8082 }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + API_WEB3_JSON_RPC_HTTP_PORT="3050" + API_WEB3_JSON_RPC_HTTP_URL="http://127.0.0.1:3050" + API_WEB3_JSON_RPC_WS_PORT="3051" + API_WEB3_JSON_RPC_WS_URL="ws://127.0.0.1:3051" + API_WEB3_JSON_RPC_REQ_ENTITIES_LIMIT=10000 + API_WEB3_JSON_RPC_FILTERS_LIMIT=10000 + API_WEB3_JSON_RPC_SUBSCRIPTIONS_LIMIT=10000 + API_WEB3_JSON_RPC_PUBSUB_POLLING_INTERVAL=200 + API_WEB3_JSON_RPC_THREADS_PER_SERVER=128 + API_WEB3_JSON_RPC_MAX_NONCE_AHEAD=5 + API_WEB3_JSON_RPC_GAS_PRICE_SCALE_FACTOR=1.2 + API_WEB3_JSON_RPC_TRANSACTIONS_PER_SEC_LIMIT=1000 + API_WEB3_JSON_RPC_REQUEST_TIMEOUT=10 + API_WEB3_JSON_RPC_ACCOUNT_PKS="0x0000000000000000000000000000000000000000000000000000000000000001,0x0000000000000000000000000000000000000000000000000000000000000002" + API_WEB3_JSON_RPC_ESTIMATE_GAS_SCALE_FACTOR=1.0 + API_WEB3_JSON_RPC_ESTIMATE_GAS_ACCEPTABLE_OVERESTIMATION=1000 + API_WEB3_JSON_RPC_MAX_TX_SIZE=1000000 + API_WEB3_JSON_RPC_VM_CONCURRENCY_LIMIT=512 + API_WEB3_JSON_RPC_FACTORY_DEPS_CACHE_SIZE_MB=128 + API_WEB3_JSON_RPC_INITIAL_WRITES_CACHE_SIZE_MB=32 + API_WEB3_JSON_RPC_LATEST_VALUES_CACHE_SIZE_MB=256 + API_WEB3_JSON_RPC_HTTP_THREADS=128 + API_WEB3_JSON_RPC_WS_THREADS=256 + API_WEB3_JSON_RPC_FEE_HISTORY_LIMIT=100 + API_WEB3_JSON_RPC_MAX_BATCH_REQUEST_SIZE=200 + API_WEB3_JSON_RPC_WEBSOCKET_REQUESTS_PER_MINUTE_LIMIT=10 + API_CONTRACT_VERIFICATION_PORT="3070" + API_CONTRACT_VERIFICATION_URL="http://127.0.0.1:3070" + API_CONTRACT_VERIFICATION_THREADS_PER_SERVER=128 + API_WEB3_JSON_RPC_MAX_RESPONSE_BODY_SIZE_MB=10 + API_PROMETHEUS_LISTENER_PORT="3312" + API_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + API_PROMETHEUS_PUSH_INTERVAL_MS=100 + API_HEALTHCHECK_PORT=8081 + API_MERKLE_TREE_PORT=8082 + "#; + lock.set_env(config); + + let actual = ApiConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/chain.rs b/core/lib/env_config/src/chain.rs new file mode 100644 index 000000000000..e64ba3c36b89 --- /dev/null +++ b/core/lib/env_config/src/chain.rs @@ -0,0 +1,151 @@ +use crate::{envy_load, FromEnv}; +use anyhow::Context as _; +use zksync_config::configs::chain::{ + ChainConfig, CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, +}; + +impl FromEnv for ChainConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + network: NetworkConfig::from_env().context("NetworkConfig")?, + state_keeper: StateKeeperConfig::from_env().context("StateKeeperConfig")?, + operations_manager: OperationsManagerConfig::from_env() + .context("OperationsManagerConfig")?, + mempool: MempoolConfig::from_env().context("MempoolConfig")?, + circuit_breaker: CircuitBreakerConfig::from_env().context("CircuitBreakerConfig")?, + }) + } +} + +impl FromEnv for NetworkConfig { + fn from_env() -> anyhow::Result { + envy_load("network", "CHAIN_ETH_") + } +} + +impl FromEnv for StateKeeperConfig { + fn from_env() -> anyhow::Result { + envy_load("state_keeper", "CHAIN_STATE_KEEPER_") + } +} + +impl FromEnv for OperationsManagerConfig { + fn from_env() -> anyhow::Result { + envy_load("operations_manager", "CHAIN_OPERATIONS_MANAGER_") + } +} + +impl FromEnv for CircuitBreakerConfig { + fn from_env() -> anyhow::Result { + envy_load("circuit_breaker", "CHAIN_CIRCUIT_BREAKER_") + } +} + +impl FromEnv for MempoolConfig { + fn from_env() -> anyhow::Result { + envy_load("mempool", "CHAIN_MEMPOOL_") + } +} + +#[cfg(test)] +mod tests { + use zksync_basic_types::L2ChainId; + + use super::*; + use crate::test_utils::{addr, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ChainConfig { + ChainConfig { + network: NetworkConfig { + network: "localhost".parse().unwrap(), + zksync_network: "localhost".to_string(), + zksync_network_id: L2ChainId::from(270), + }, + state_keeper: StateKeeperConfig { + transaction_slots: 50, + block_commit_deadline_ms: 2500, + miniblock_commit_deadline_ms: 1000, + miniblock_seal_queue_capacity: 10, + max_single_tx_gas: 1_000_000, + max_allowed_l2_tx_gas_limit: 2_000_000_000, + close_block_at_eth_params_percentage: 0.2, + close_block_at_gas_percentage: 0.8, + close_block_at_geometry_percentage: 0.5, + reject_tx_at_eth_params_percentage: 0.8, + reject_tx_at_geometry_percentage: 0.3, + fee_account_addr: addr("de03a0B5963f75f1C8485B355fF6D30f3093BDE7"), + reject_tx_at_gas_percentage: 0.5, + fair_l2_gas_price: 250000000, + validation_computational_gas_limit: 10_000_000, + save_call_traces: false, + virtual_blocks_interval: 1, + virtual_blocks_per_miniblock: 1, + upload_witness_inputs_to_gcs: false, + enum_index_migration_chunk_size: Some(2_000), + }, + operations_manager: OperationsManagerConfig { + delay_interval: 100, + }, + mempool: MempoolConfig { + sync_interval_ms: 10, + sync_batch_size: 1000, + capacity: 1_000_000, + stuck_tx_timeout: 10, + remove_stuck_txs: true, + delay_interval: 100, + }, + circuit_breaker: CircuitBreakerConfig { + sync_interval_ms: 1000, + http_req_max_retry_number: 5, + http_req_retry_interval_sec: 2, + replication_lag_limit_sec: Some(10), + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CHAIN_ETH_NETWORK="localhost" + CHAIN_ETH_ZKSYNC_NETWORK="localhost" + CHAIN_ETH_ZKSYNC_NETWORK_ID=270 + CHAIN_STATE_KEEPER_TRANSACTION_SLOTS="50" + CHAIN_STATE_KEEPER_FEE_ACCOUNT_ADDR="0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7" + CHAIN_STATE_KEEPER_MAX_SINGLE_TX_GAS="1000000" + CHAIN_STATE_KEEPER_MAX_ALLOWED_L2_TX_GAS_LIMIT="2000000000" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GEOMETRY_PERCENTAGE="0.5" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_GAS_PERCENTAGE="0.8" + CHAIN_STATE_KEEPER_CLOSE_BLOCK_AT_ETH_PARAMS_PERCENTAGE="0.2" + CHAIN_STATE_KEEPER_REJECT_TX_AT_GEOMETRY_PERCENTAGE="0.3" + CHAIN_STATE_KEEPER_REJECT_TX_AT_ETH_PARAMS_PERCENTAGE="0.8" + CHAIN_STATE_KEEPER_REJECT_TX_AT_GAS_PERCENTAGE="0.5" + CHAIN_STATE_KEEPER_BLOCK_COMMIT_DEADLINE_MS="2500" + CHAIN_STATE_KEEPER_MINIBLOCK_COMMIT_DEADLINE_MS="1000" + CHAIN_STATE_KEEPER_MINIBLOCK_SEAL_QUEUE_CAPACITY="10" + CHAIN_STATE_KEEPER_FAIR_L2_GAS_PRICE="250000000" + CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT="10000000" + CHAIN_STATE_KEEPER_SAVE_CALL_TRACES="false" + CHAIN_STATE_KEEPER_UPLOAD_WITNESS_INPUTS_TO_GCS="false" + CHAIN_STATE_KEEPER_ENUM_INDEX_MIGRATION_CHUNK_SIZE="2000" + CHAIN_OPERATIONS_MANAGER_DELAY_INTERVAL="100" + CHAIN_MEMPOOL_SYNC_INTERVAL_MS="10" + CHAIN_MEMPOOL_SYNC_BATCH_SIZE="1000" + CHAIN_MEMPOOL_STUCK_TX_TIMEOUT="10" + CHAIN_MEMPOOL_REMOVE_STUCK_TXS="true" + CHAIN_MEMPOOL_DELAY_INTERVAL="100" + CHAIN_MEMPOOL_CAPACITY="1000000" + CHAIN_CIRCUIT_BREAKER_SYNC_INTERVAL_MS="1000" + CHAIN_CIRCUIT_BREAKER_HTTP_REQ_MAX_RETRY_NUMBER="5" + CHAIN_CIRCUIT_BREAKER_HTTP_REQ_RETRY_INTERVAL_SEC="2" + CHAIN_CIRCUIT_BREAKER_REPLICATION_LAG_LIMIT_SEC="10" + "#; + lock.set_env(config); + + let actual = ChainConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/circuit_synthesizer.rs b/core/lib/env_config/src/circuit_synthesizer.rs new file mode 100644 index 000000000000..a59e1adcd656 --- /dev/null +++ b/core/lib/env_config/src/circuit_synthesizer.rs @@ -0,0 +1,51 @@ +use zksync_config::configs::CircuitSynthesizerConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for CircuitSynthesizerConfig { + fn from_env() -> anyhow::Result { + envy_load("circuit_synthesizer", "CIRCUIT_SYNTHESIZER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> CircuitSynthesizerConfig { + CircuitSynthesizerConfig { + generation_timeout_in_secs: 1000u16, + max_attempts: 2, + gpu_prover_queue_timeout_in_secs: 1000u16, + prover_instance_wait_timeout_in_secs: 1000u16, + prover_instance_poll_time_in_milli_secs: 250u16, + prometheus_listener_port: 3314, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + prover_group_id: 0, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CIRCUIT_SYNTHESIZER_GENERATION_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_MAX_ATTEMPTS=2 + CIRCUIT_SYNTHESIZER_GPU_PROVER_QUEUE_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 + CIRCUIT_SYNTHESIZER_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 + CIRCUIT_SYNTHESIZER_PROMETHEUS_LISTENER_PORT=3314 + CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + CIRCUIT_SYNTHESIZER_PROMETHEUS_PUSH_INTERVAL_MS=100 + CIRCUIT_SYNTHESIZER_PROVER_GROUP_ID=0 + "#; + lock.set_env(config); + + let actual = CircuitSynthesizerConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/contract_verifier.rs b/core/lib/env_config/src/contract_verifier.rs new file mode 100644 index 000000000000..88c462cef2df --- /dev/null +++ b/core/lib/env_config/src/contract_verifier.rs @@ -0,0 +1,39 @@ +use zksync_config::ContractVerifierConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ContractVerifierConfig { + fn from_env() -> anyhow::Result { + envy_load("contract_verifier", "CONTRACT_VERIFIER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ContractVerifierConfig { + ContractVerifierConfig { + compilation_timeout: 30, + polling_interval: Some(1000), + prometheus_port: 3314, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30 + CONTRACT_VERIFIER_POLLING_INTERVAL=1000 + CONTRACT_VERIFIER_PROMETHEUS_PORT=3314 + "#; + lock.set_env(config); + + let actual = ContractVerifierConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/contracts.rs b/core/lib/env_config/src/contracts.rs new file mode 100644 index 000000000000..8c58483db064 --- /dev/null +++ b/core/lib/env_config/src/contracts.rs @@ -0,0 +1,108 @@ +use zksync_config::ContractsConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ContractsConfig { + fn from_env() -> anyhow::Result { + envy_load("contracts", "CONTRACTS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::{addr, hash, EnvMutex}; + use zksync_config::configs::contracts::ProverAtGenesis; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ContractsConfig { + ContractsConfig { + governance_addr: addr("d8dA6BF26964aF9D7eEd9e03E53415D37aA96045"), + mailbox_facet_addr: addr("0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd"), + executor_facet_addr: addr("18B631537801963A964211C0E86645c1aBfbB2d3"), + admin_facet_addr: addr("1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32"), + getters_facet_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + verifier_addr: addr("34782eE00206EAB6478F2692caa800e4A581687b"), + diamond_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), + diamond_upgrade_init_addr: addr("FFC35A5e767BE36057c34586303498e3de7C62Ba"), + diamond_proxy_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), + validator_timelock_addr: addr("F00B988a98Ca742e7958DeF9F7823b5908715f4a"), + genesis_tx_hash: hash( + "b99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e", + ), + l1_erc20_bridge_proxy_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_erc20_bridge_impl_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l2_erc20_bridge_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_allow_list_addr: addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888"), + l1_weth_bridge_proxy_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), + l2_weth_bridge_addr: Some(addr("8656770FA78c830456B00B4fFCeE6b1De0e1b888")), + l2_testnet_paymaster_addr: Some(addr("FC073319977e314F251EAE6ae6bE76B0B3BAeeCF")), + recursion_scheduler_level_vk_hash: hash( + "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", + ), + recursion_node_level_vk_hash: hash( + "0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8", + ), + recursion_leaf_level_vk_hash: hash( + "0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210", + ), + recursion_circuits_set_vks_hash: hash( + "0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0", + ), + l1_multicall3_addr: addr("0xcA11bde05977b3631167028862bE2a173976CA11"), + fri_recursion_scheduler_level_vk_hash: hash( + "0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d", + ), + fri_recursion_node_level_vk_hash: hash( + "0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080", + ), + fri_recursion_leaf_level_vk_hash: hash( + "0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828", + ), + prover_at_genesis: ProverAtGenesis::Fri, + snark_wrapper_vk_hash: hash( + "0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2", + ), + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" +CONTRACTS_GOVERNANCE_ADDR="0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045" +CONTRACTS_MAILBOX_FACET_ADDR="0x0f6Fa881EF414Fc6E818180657c2d5CD7Ac6cCAd" +CONTRACTS_EXECUTOR_FACET_ADDR="0x18B631537801963A964211C0E86645c1aBfbB2d3" +CONTRACTS_ADMIN_FACET_ADDR="0x1e12b20BE86bEc3A0aC95aA52ade345cB9AE7a32" +CONTRACTS_GETTERS_FACET_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_VERIFIER_ADDR="0x34782eE00206EAB6478F2692caa800e4A581687b" +CONTRACTS_DIAMOND_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" +CONTRACTS_DIAMOND_UPGRADE_INIT_ADDR="0xFFC35A5e767BE36057c34586303498e3de7C62Ba" +CONTRACTS_DIAMOND_PROXY_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" +CONTRACTS_VALIDATOR_TIMELOCK_ADDR="0xF00B988a98Ca742e7958DeF9F7823b5908715f4a" +CONTRACTS_GENESIS_TX_HASH="0xb99ebfea46cbe05a21cd80fe5597d97b204befc52a16303f579c607dc1ac2e2e" +CONTRACTS_L1_ERC20_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_ALLOW_LIST_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_ERC20_BRIDGE_IMPL_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_ERC20_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L1_WETH_BRIDGE_PROXY_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_WETH_BRIDGE_ADDR="0x8656770FA78c830456B00B4fFCeE6b1De0e1b888" +CONTRACTS_L2_TESTNET_PAYMASTER_ADDR="FC073319977e314F251EAE6ae6bE76B0B3BAeeCF" +CONTRACTS_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" +CONTRACTS_RECURSION_NODE_LEVEL_VK_HASH="0x1186ec268d49f1905f8d9c1e9d39fc33e98c74f91d91a21b8f7ef78bd09a8db8" +CONTRACTS_RECURSION_LEAF_LEVEL_VK_HASH="0x101e08b00193e529145ee09823378ef51a3bc8966504064f1f6ba3f1ba863210" +CONTRACTS_RECURSION_CIRCUITS_SET_VKS_HASH="0x142a364ef2073132eaf07aa7f3d8495065be5b92a2dc14fda09b4216affed9c0" +CONTRACTS_L1_MULTICALL3_ADDR="0xcA11bde05977b3631167028862bE2a173976CA11" +CONTRACTS_FRI_RECURSION_SCHEDULER_LEVEL_VK_HASH="0x201d4c7d8e781d51a3bbd451a43a8f45240bb765b565ae6ce69192d918c3563d" +CONTRACTS_FRI_RECURSION_NODE_LEVEL_VK_HASH="0x5a3ef282b21e12fe1f4438e5bb158fc5060b160559c5158c6389d62d9fe3d080" +CONTRACTS_FRI_RECURSION_LEAF_LEVEL_VK_HASH="0x72167c43a46cf38875b267d67716edc4563861364a3c03ab7aee73498421e828" +CONTRACTS_PROVER_AT_GENESIS="fri" +CONTRACTS_SNARK_WRAPPER_VK_HASH="0x4be443afd605a782b6e56d199df2460a025c81b3dea144e135bece83612563f2" + "#; + lock.set_env(config); + + let actual = ContractsConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/database.rs b/core/lib/env_config/src/database.rs new file mode 100644 index 000000000000..eeb67205d4de --- /dev/null +++ b/core/lib/env_config/src/database.rs @@ -0,0 +1,100 @@ +use zksync_config::DBConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for DBConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + merkle_tree: envy_load("database_merkle_tree", "DATABASE_MERKLE_TREE_")?, + ..envy_load("database", "DATABASE_")? + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::database::MerkleTreeMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + DATABASE_STATE_KEEPER_DB_PATH="/db/state_keeper" + DATABASE_MERKLE_TREE_BACKUP_PATH="/db/backups" + DATABASE_MERKLE_TREE_PATH="/db/tree" + DATABASE_MERKLE_TREE_MODE=lightweight + DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=250 + DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB=512 + DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC=60 + DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50 + DATABASE_BACKUP_COUNT=5 + DATABASE_BACKUP_INTERVAL_MS=60000 + "#; + lock.set_env(config); + + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.state_keeper_db_path, "/db/state_keeper"); + assert_eq!(db_config.merkle_tree.path, "/db/tree"); + assert_eq!(db_config.merkle_tree.backup_path, "/db/backups"); + assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Lightweight); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 250); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); + assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 512); + assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 60); + assert_eq!(db_config.backup_count, 5); + assert_eq!(db_config.backup_interval().as_secs(), 60); + } + + #[test] + fn from_empty_env() { + let mut lock = MUTEX.lock(); + lock.remove_env(&[ + "DATABASE_STATE_KEEPER_DB_PATH", + "DATABASE_MERKLE_TREE_BACKUP_PATH", + "DATABASE_MERKLE_TREE_PATH", + "DATABASE_MERKLE_TREE_MODE", + "DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE", + "DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB", + "DATABASE_MERKLE_TREE_MEMTABLE_CAPACITY_MB", + "DATABASE_MERKLE_TREE_STALLED_WRITES_TIMEOUT_SEC", + "DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER", + "DATABASE_BACKUP_COUNT", + "DATABASE_BACKUP_INTERVAL_MS", + ]); + + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.state_keeper_db_path, "./db/state_keeper"); + assert_eq!(db_config.merkle_tree.path, "./db/lightweight-new"); + assert_eq!(db_config.merkle_tree.backup_path, "./db/backups"); + assert_eq!(db_config.merkle_tree.mode, MerkleTreeMode::Full); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 500); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 20); + assert_eq!(db_config.merkle_tree.block_cache_size_mb, 128); + assert_eq!(db_config.merkle_tree.memtable_capacity_mb, 256); + assert_eq!(db_config.merkle_tree.stalled_writes_timeout_sec, 30); + assert_eq!(db_config.backup_count, 5); + assert_eq!(db_config.backup_interval().as_secs(), 60); + + // Check that new env variable for Merkle tree path is supported + lock.set_env("DATABASE_MERKLE_TREE_PATH=/db/tree/main"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.path, "/db/tree/main"); + + lock.set_env("DATABASE_MERKLE_TREE_MULTI_GET_CHUNK_SIZE=200"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.multi_get_chunk_size, 200); + + lock.set_env("DATABASE_MERKLE_TREE_BLOCK_CACHE_SIZE_MB=256"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.block_cache_size_mb, 256); + + lock.set_env("DATABASE_MERKLE_TREE_MAX_L1_BATCHES_PER_ITER=50"); + let db_config = DBConfig::from_env().unwrap(); + assert_eq!(db_config.merkle_tree.max_l1_batches_per_iter, 50); + } +} diff --git a/core/lib/env_config/src/eth_client.rs b/core/lib/env_config/src/eth_client.rs new file mode 100644 index 000000000000..378ca84b0e2d --- /dev/null +++ b/core/lib/env_config/src/eth_client.rs @@ -0,0 +1,44 @@ +use zksync_config::ETHClientConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHClientConfig { + fn from_env() -> anyhow::Result { + let config: Self = envy_load("eth_client", "ETH_CLIENT_")?; + if config.web3_url.find(',').is_some() { + anyhow::bail!( + "Multiple web3 URLs aren't supported anymore. Provided invalid value: {}", + config.web3_url + ); + } + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHClientConfig { + ETHClientConfig { + chain_id: 9, + web3_url: "http://127.0.0.1:8545".into(), + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_CLIENT_CHAIN_ID="9" + ETH_CLIENT_WEB3_URL="http://127.0.0.1:8545" + "#; + lock.set_env(config); + + let actual = ETHClientConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/eth_sender.rs b/core/lib/env_config/src/eth_sender.rs new file mode 100644 index 000000000000..f0ee6030b029 --- /dev/null +++ b/core/lib/env_config/src/eth_sender.rs @@ -0,0 +1,111 @@ +use anyhow::Context as _; +use zksync_config::{configs::eth_sender::SenderConfig, ETHSenderConfig, GasAdjusterConfig}; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHSenderConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + sender: SenderConfig::from_env().context("SenderConfig")?, + gas_adjuster: GasAdjusterConfig::from_env().context("GasAdjusterConfig")?, + }) + } +} + +impl FromEnv for SenderConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_sender", "ETH_SENDER_SENDER_") + } +} + +impl FromEnv for GasAdjusterConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_sender.gas_adjuster", "ETH_SENDER_GAS_ADJUSTER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::eth_sender::{ProofLoadingMode, ProofSendingMode}; + + use super::*; + use crate::test_utils::{hash, EnvMutex}; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHSenderConfig { + ETHSenderConfig { + sender: SenderConfig { + aggregated_proof_sizes: vec![1, 5], + aggregated_block_commit_deadline: 30, + aggregated_block_prove_deadline: 3_000, + aggregated_block_execute_deadline: 4_000, + max_aggregated_tx_gas: 4_000_000, + max_eth_tx_data_size: 120_000, + + timestamp_criteria_max_allowed_lag: 30, + max_aggregated_blocks_to_commit: 3, + max_aggregated_blocks_to_execute: 4, + wait_confirmations: Some(1), + tx_poll_period: 3, + aggregate_tx_poll_period: 3, + max_txs_in_flight: 3, + proof_sending_mode: ProofSendingMode::SkipEveryProof, + l1_batch_min_age_before_execute_seconds: Some(1000), + max_acceptable_priority_fee_in_gwei: 100_000_000_000, + proof_loading_mode: ProofLoadingMode::OldProofFromDb, + }, + gas_adjuster: GasAdjusterConfig { + default_priority_fee_per_gas: 20000000000, + max_base_fee_samples: 10000, + pricing_formula_parameter_a: 1.5, + pricing_formula_parameter_b: 1.0005, + internal_l1_pricing_multiplier: 0.8, + internal_enforced_l1_gas_price: None, + poll_period: 15, + max_l1_gas_price: Some(100000000), + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_SENDER_SENDER_WAIT_CONFIRMATIONS="1" + ETH_SENDER_SENDER_TX_POLL_PERIOD="3" + ETH_SENDER_SENDER_AGGREGATE_TX_POLL_PERIOD="3" + ETH_SENDER_SENDER_MAX_TXS_IN_FLIGHT="3" + ETH_SENDER_SENDER_OPERATOR_PRIVATE_KEY="0x27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be" + ETH_SENDER_SENDER_PROOF_SENDING_MODE="SkipEveryProof" + ETH_SENDER_GAS_ADJUSTER_DEFAULT_PRIORITY_FEE_PER_GAS="20000000000" + ETH_SENDER_GAS_ADJUSTER_MAX_BASE_FEE_SAMPLES="10000" + ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_A="1.5" + ETH_SENDER_GAS_ADJUSTER_PRICING_FORMULA_PARAMETER_B="1.0005" + ETH_SENDER_GAS_ADJUSTER_INTERNAL_L1_PRICING_MULTIPLIER="0.8" + ETH_SENDER_GAS_ADJUSTER_POLL_PERIOD="15" + ETH_SENDER_GAS_ADJUSTER_MAX_L1_GAS_PRICE="100000000" + ETH_SENDER_WAIT_FOR_PROOFS="false" + ETH_SENDER_SENDER_AGGREGATED_PROOF_SIZES="1,5" + ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_COMMIT="3" + ETH_SENDER_SENDER_MAX_AGGREGATED_BLOCKS_TO_EXECUTE="4" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_COMMIT_DEADLINE="30" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_PROVE_DEADLINE="3000" + ETH_SENDER_SENDER_AGGREGATED_BLOCK_EXECUTE_DEADLINE="4000" + ETH_SENDER_SENDER_TIMESTAMP_CRITERIA_MAX_ALLOWED_LAG="30" + ETH_SENDER_SENDER_MAX_AGGREGATED_TX_GAS="4000000" + ETH_SENDER_SENDER_MAX_ETH_TX_DATA_SIZE="120000" + ETH_SENDER_SENDER_L1_BATCH_MIN_AGE_BEFORE_EXECUTE_SECONDS="1000" + ETH_SENDER_SENDER_MAX_ACCEPTABLE_PRIORITY_FEE_IN_GWEI="100000000000" + ETH_SENDER_SENDER_PROOF_LOADING_MODE="OldProofFromDb" + "#; + lock.set_env(config); + + let actual = ETHSenderConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + assert_eq!( + actual.sender.private_key().unwrap(), + hash("27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be") + ); + } +} diff --git a/core/lib/env_config/src/eth_watch.rs b/core/lib/env_config/src/eth_watch.rs new file mode 100644 index 000000000000..ef90b7e9c46c --- /dev/null +++ b/core/lib/env_config/src/eth_watch.rs @@ -0,0 +1,37 @@ +use zksync_config::ETHWatchConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ETHWatchConfig { + fn from_env() -> anyhow::Result { + envy_load("eth_watch", "ETH_WATCH_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ETHWatchConfig { + ETHWatchConfig { + confirmations_for_eth_event: Some(0), + eth_node_poll_interval: 300, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + ETH_WATCH_CONFIRMATIONS_FOR_ETH_EVENT="0" + ETH_WATCH_ETH_NODE_POLL_INTERVAL="300" + "#; + lock.set_env(config); + + let actual = ETHWatchConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fetcher.rs b/core/lib/env_config/src/fetcher.rs new file mode 100644 index 000000000000..4f86488b2c41 --- /dev/null +++ b/core/lib/env_config/src/fetcher.rs @@ -0,0 +1,68 @@ +use zksync_config::FetcherConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FetcherConfig { + fn from_env() -> anyhow::Result { + Ok(Self { + token_list: envy_load("token_list", "FETCHER_TOKEN_LIST_")?, + token_price: envy_load("token_price", "FETCHER_TOKEN_PRICE_")?, + token_trading_volume: envy_load( + "token_trading_volume", + "FETCHER_TOKEN_TRADING_VOLUME_", + )?, + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::fetcher::{ + SingleFetcherConfig, TokenListSource, TokenPriceSource, TokenTradingVolumeSource, + }; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FetcherConfig { + FetcherConfig { + token_list: SingleFetcherConfig { + source: TokenListSource::OneInch, + url: "http://127.0.0.1:1020".into(), + fetching_interval: 10, + }, + token_price: SingleFetcherConfig { + source: TokenPriceSource::CoinGecko, + url: "http://127.0.0.1:9876".into(), + fetching_interval: 7, + }, + token_trading_volume: SingleFetcherConfig { + source: TokenTradingVolumeSource::Uniswap, + url: "http://127.0.0.1:9975/graphql".to_string(), + fetching_interval: 5, + }, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FETCHER_TOKEN_LIST_SOURCE="OneInch" + FETCHER_TOKEN_LIST_URL="http://127.0.0.1:1020" + FETCHER_TOKEN_LIST_FETCHING_INTERVAL="10" + FETCHER_TOKEN_PRICE_SOURCE="CoinGecko" + FETCHER_TOKEN_PRICE_URL="http://127.0.0.1:9876" + FETCHER_TOKEN_PRICE_FETCHING_INTERVAL="7" + FETCHER_TOKEN_TRADING_VOLUME_SOURCE="Uniswap" + FETCHER_TOKEN_TRADING_VOLUME_URL="http://127.0.0.1:9975/graphql" + FETCHER_TOKEN_TRADING_VOLUME_FETCHING_INTERVAL="5" + "#; + lock.set_env(config); + + let actual = FetcherConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_proof_compressor.rs b/core/lib/env_config/src/fri_proof_compressor.rs new file mode 100644 index 000000000000..2594433025e4 --- /dev/null +++ b/core/lib/env_config/src/fri_proof_compressor.rs @@ -0,0 +1,54 @@ +use zksync_config::configs::FriProofCompressorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProofCompressorConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_proof_compressor", "FRI_PROOF_COMPRESSOR_") + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::EnvMutex; + + use super::*; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProofCompressorConfig { + FriProofCompressorConfig { + compression_mode: 1, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + generation_timeout_in_secs: 3000, + max_attempts: 5, + universal_setup_path: "keys/setup/setup_2^26.key".to_string(), + universal_setup_download_url: + "https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" + .to_string(), + verify_wrapper_proof: false, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_PROOF_COMPRESSOR_COMPRESSION_MODE=1 + FRI_PROOF_COMPRESSOR_PROMETHEUS_LISTENER_PORT=3316 + FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_PROOF_COMPRESSOR_PROMETHEUS_PUSH_INTERVAL_MS=100 + FRI_PROOF_COMPRESSOR_GENERATION_TIMEOUT_IN_SECS=3000 + FRI_PROOF_COMPRESSOR_MAX_ATTEMPTS=5 + FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_PATH="keys/setup/setup_2^26.key" + FRI_PROOF_COMPRESSOR_UNIVERSAL_SETUP_DOWNLOAD_URL="https://storage.googleapis.com/matterlabs-setup-keys-us/setup-keys/setup_2^26.key" + FRI_PROOF_COMPRESSOR_VERIFY_WRAPPER_PROOF=false + "#; + lock.set_env(config); + + let actual = FriProofCompressorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover.rs b/core/lib/env_config/src/fri_prover.rs new file mode 100644 index 000000000000..200f22d89b7e --- /dev/null +++ b/core/lib/env_config/src/fri_prover.rs @@ -0,0 +1,59 @@ +use zksync_config::configs::FriProverConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProverConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_prover", "FRI_PROVER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::fri_prover::SetupLoadMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProverConfig { + FriProverConfig { + setup_data_path: "/usr/src/setup-data".to_string(), + prometheus_port: 3315, + max_attempts: 10, + generation_timeout_in_secs: 300, + base_layer_circuit_ids_to_be_verified: vec![1, 5], + recursive_layer_circuit_ids_to_be_verified: vec![1, 2, 3], + setup_load_mode: SetupLoadMode::FromDisk, + specialized_group_id: 10, + witness_vector_generator_thread_count: Some(5), + queue_capacity: 10, + witness_vector_receiver_port: 3316, + shall_save_to_public_bucket: true, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_PROVER_SETUP_DATA_PATH="/usr/src/setup-data" + FRI_PROVER_PROMETHEUS_PORT="3315" + FRI_PROVER_MAX_ATTEMPTS="10" + FRI_PROVER_GENERATION_TIMEOUT_IN_SECS="300" + FRI_PROVER_BASE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,5" + FRI_PROVER_RECURSIVE_LAYER_CIRCUIT_IDS_TO_BE_VERIFIED="1,2,3" + FRI_PROVER_SETUP_LOAD_MODE="FromDisk" + FRI_PROVER_SPECIALIZED_GROUP_ID="10" + FRI_PROVER_WITNESS_VECTOR_GENERATOR_THREAD_COUNT="5" + FRI_PROVER_QUEUE_CAPACITY="10" + FRI_PROVER_WITNESS_VECTOR_RECEIVER_PORT="3316" + FRI_PROVER_SHALL_SAVE_TO_PUBLIC_BUCKET=true + "#; + lock.set_env(config); + + let actual = FriProverConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover_gateway.rs b/core/lib/env_config/src/fri_prover_gateway.rs new file mode 100644 index 000000000000..cfb2622bb12a --- /dev/null +++ b/core/lib/env_config/src/fri_prover_gateway.rs @@ -0,0 +1,42 @@ +use zksync_config::configs::FriProverGatewayConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriProverGatewayConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_prover_gateway", "FRI_PROVER_GATEWAY_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriProverGatewayConfig { + FriProverGatewayConfig { + api_url: "http://private-dns-for-server".to_string(), + api_poll_duration_secs: 100, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + } + } + + #[test] + fn from_env() { + let config = r#" + FRI_PROVER_GATEWAY_API_URL="http://private-dns-for-server" + FRI_PROVER_GATEWAY_API_POLL_DURATION_SECS="100" + FRI_PROVER_GATEWAY_PROMETHEUS_LISTENER_PORT=3316 + FRI_PROVER_GATEWAY_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_PROVER_GATEWAY_PROMETHEUS_PUSH_INTERVAL_MS=100 + "#; + let mut lock = MUTEX.lock(); + lock.set_env(config); + let actual = FriProverGatewayConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_prover_group.rs b/core/lib/env_config/src/fri_prover_group.rs new file mode 100644 index 000000000000..373febb6abb2 --- /dev/null +++ b/core/lib/env_config/src/fri_prover_group.rs @@ -0,0 +1,247 @@ +use std::{ + collections::{HashMap, HashSet}, + env, +}; + +use zksync_basic_types::basic_fri_types::CircuitIdRoundTuple; +use zksync_config::configs::fri_prover_group::FriProverGroupConfig; + +use crate::FromEnv; + +fn load_from_env_variable() -> HashMap> { + // Prepare a hash map to store the mapping of group to a vector of tuples + let mut groups: HashMap> = (0..=12) + .map(|i| (format!("group_{}", i), HashSet::new())) + .collect(); + + // Separate environment variables into Circuit Id and Aggregation Round + let mut circuit_ids = HashMap::new(); + let mut aggregation_rounds = HashMap::new(); + for (key, value) in env::vars() { + if key.contains("_CIRCUIT_ID") { + circuit_ids.insert(key, value); + } else if key.contains("_AGGREGATION_ROUND") { + aggregation_rounds.insert(key, value); + } + } + + // Iterate over all circuit id variables + for (key, value_str) in circuit_ids { + let key_parts: Vec<&str> = key.split('_').collect(); + if let (Some(group_key), Some(value), Some(index_str)) = ( + key_parts.get(4), + value_str.parse::().ok(), + key_parts.get(5), + ) { + let round_key = format!( + "FRI_PROVER_GROUP_GROUP_{}_{}_AGGREGATION_ROUND", + group_key, index_str + ); + if let Some(round_str) = aggregation_rounds.get(&round_key) { + if let Ok(round) = round_str.parse::() { + let tuple = CircuitIdRoundTuple::new(value, round); + if let Some(group) = groups.get_mut(&format!("group_{}", group_key)) { + group.insert(tuple); + } + } + } + } + } + groups +} + +impl FromEnv for FriProverGroupConfig { + fn from_env() -> anyhow::Result { + let mut groups = load_from_env_variable(); + let config = FriProverGroupConfig { + group_0: groups.remove("group_0").unwrap_or_default(), + group_1: groups.remove("group_1").unwrap_or_default(), + group_2: groups.remove("group_2").unwrap_or_default(), + group_3: groups.remove("group_3").unwrap_or_default(), + group_4: groups.remove("group_4").unwrap_or_default(), + group_5: groups.remove("group_5").unwrap_or_default(), + group_6: groups.remove("group_6").unwrap_or_default(), + group_7: groups.remove("group_7").unwrap_or_default(), + group_8: groups.remove("group_8").unwrap_or_default(), + group_9: groups.remove("group_9").unwrap_or_default(), + group_10: groups.remove("group_10").unwrap_or_default(), + group_11: groups.remove("group_11").unwrap_or_default(), + group_12: groups.remove("group_12").unwrap_or_default(), + }; + config.validate(); + Ok(config) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use super::*; + + fn expected_config() -> FriProverGroupConfig { + FriProverGroupConfig { + group_0: vec![ + CircuitIdRoundTuple::new(1, 3), + CircuitIdRoundTuple::new(2, 2), + ] + .into_iter() + .collect::>(), + group_1: vec![CircuitIdRoundTuple::new(1, 0)] + .into_iter() + .collect::>(), + group_2: vec![ + CircuitIdRoundTuple::new(2, 0), + CircuitIdRoundTuple::new(4, 0), + CircuitIdRoundTuple::new(6, 0), + CircuitIdRoundTuple::new(9, 0), + ] + .into_iter() + .collect::>(), + group_3: vec![CircuitIdRoundTuple::new(3, 0)] + .into_iter() + .collect::>(), + group_4: vec![ + CircuitIdRoundTuple::new(11, 0), + CircuitIdRoundTuple::new(12, 0), + CircuitIdRoundTuple::new(13, 0), + ] + .into_iter() + .collect::>(), + group_5: vec![CircuitIdRoundTuple::new(5, 0)] + .into_iter() + .collect::>(), + group_6: vec![CircuitIdRoundTuple::new(3, 1)] + .into_iter() + .collect::>(), + group_7: vec![CircuitIdRoundTuple::new(7, 0)] + .into_iter() + .collect::>(), + group_8: vec![CircuitIdRoundTuple::new(8, 0)] + .into_iter() + .collect::>(), + group_9: vec![ + CircuitIdRoundTuple::new(12, 1), + CircuitIdRoundTuple::new(13, 1), + CircuitIdRoundTuple::new(14, 1), + CircuitIdRoundTuple::new(15, 1), + ] + .into_iter() + .collect::>(), + group_10: vec![CircuitIdRoundTuple::new(10, 0)] + .into_iter() + .collect::>(), + group_11: vec![ + CircuitIdRoundTuple::new(7, 1), + CircuitIdRoundTuple::new(8, 1), + CircuitIdRoundTuple::new(10, 1), + CircuitIdRoundTuple::new(11, 1), + ] + .into_iter() + .collect::>(), + group_12: vec![ + CircuitIdRoundTuple::new(4, 1), + CircuitIdRoundTuple::new(5, 1), + CircuitIdRoundTuple::new(6, 1), + CircuitIdRoundTuple::new(9, 1), + ] + .into_iter() + .collect::>(), + } + } + + #[test] + fn from_env() { + let groups = [ + ("FRI_PROVER_GROUP_GROUP_0_0", CircuitIdRoundTuple::new(1, 3)), + ("FRI_PROVER_GROUP_GROUP_0_1", CircuitIdRoundTuple::new(2, 2)), + ("FRI_PROVER_GROUP_GROUP_1_0", CircuitIdRoundTuple::new(1, 0)), + ("FRI_PROVER_GROUP_GROUP_2_0", CircuitIdRoundTuple::new(2, 0)), + ("FRI_PROVER_GROUP_GROUP_2_1", CircuitIdRoundTuple::new(4, 0)), + ("FRI_PROVER_GROUP_GROUP_2_2", CircuitIdRoundTuple::new(6, 0)), + ("FRI_PROVER_GROUP_GROUP_2_3", CircuitIdRoundTuple::new(9, 0)), + ("FRI_PROVER_GROUP_GROUP_3_0", CircuitIdRoundTuple::new(3, 0)), + ( + "FRI_PROVER_GROUP_GROUP_4_0", + CircuitIdRoundTuple::new(11, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_4_1", + CircuitIdRoundTuple::new(12, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_4_2", + CircuitIdRoundTuple::new(13, 0), + ), + ("FRI_PROVER_GROUP_GROUP_5_0", CircuitIdRoundTuple::new(5, 0)), + ("FRI_PROVER_GROUP_GROUP_6_0", CircuitIdRoundTuple::new(3, 1)), + ("FRI_PROVER_GROUP_GROUP_7_0", CircuitIdRoundTuple::new(7, 0)), + ("FRI_PROVER_GROUP_GROUP_8_0", CircuitIdRoundTuple::new(8, 0)), + ( + "FRI_PROVER_GROUP_GROUP_9_0", + CircuitIdRoundTuple::new(12, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_1", + CircuitIdRoundTuple::new(13, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_2", + CircuitIdRoundTuple::new(14, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_9_3", + CircuitIdRoundTuple::new(15, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_10_0", + CircuitIdRoundTuple::new(10, 0), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_0", + CircuitIdRoundTuple::new(7, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_1", + CircuitIdRoundTuple::new(8, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_2", + CircuitIdRoundTuple::new(10, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_11_3", + CircuitIdRoundTuple::new(11, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_0", + CircuitIdRoundTuple::new(4, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_1", + CircuitIdRoundTuple::new(5, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_2", + CircuitIdRoundTuple::new(6, 1), + ), + ( + "FRI_PROVER_GROUP_GROUP_12_3", + CircuitIdRoundTuple::new(9, 1), + ), + ]; + + for (key_base, circuit_round_tuple) in &groups { + let circuit_id_key = format!("{}_CIRCUIT_ID", key_base); + let aggregation_round_key = format!("{}_AGGREGATION_ROUND", key_base); + env::set_var(&circuit_id_key, circuit_round_tuple.circuit_id.to_string()); + env::set_var( + &aggregation_round_key, + circuit_round_tuple.aggregation_round.to_string(), + ); + } + + let actual = FriProverGroupConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_witness_generator.rs b/core/lib/env_config/src/fri_witness_generator.rs new file mode 100644 index 000000000000..ab3d5dc4d133 --- /dev/null +++ b/core/lib/env_config/src/fri_witness_generator.rs @@ -0,0 +1,46 @@ +use zksync_config::configs::FriWitnessGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriWitnessGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load("fri_witness", "FRI_WITNESS_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriWitnessGeneratorConfig { + FriWitnessGeneratorConfig { + generation_timeout_in_secs: 900u16, + max_attempts: 4, + blocks_proving_percentage: Some(30), + dump_arguments_for_blocks: vec![2, 3], + last_l1_batch_to_process: None, + force_process_block: Some(1), + shall_save_to_public_bucket: true, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_WITNESS_GENERATION_TIMEOUT_IN_SECS=900 + FRI_WITNESS_MAX_ATTEMPTS=4 + FRI_WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" + FRI_WITNESS_BLOCKS_PROVING_PERCENTAGE="30" + FRI_WITNESS_FORCE_PROCESS_BLOCK="1" + FRI_WITNESS_SHALL_SAVE_TO_PUBLIC_BUCKET=true + "#; + lock.set_env(config); + + let actual = FriWitnessGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/fri_witness_vector_generator.rs b/core/lib/env_config/src/fri_witness_vector_generator.rs new file mode 100644 index 000000000000..f564aaf13cec --- /dev/null +++ b/core/lib/env_config/src/fri_witness_vector_generator.rs @@ -0,0 +1,50 @@ +use zksync_config::configs::FriWitnessVectorGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for FriWitnessVectorGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load( + "fri_witness_vector_generator", + "FRI_WITNESS_VECTOR_GENERATOR_", + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> FriWitnessVectorGeneratorConfig { + FriWitnessVectorGeneratorConfig { + max_prover_reservation_duration_in_secs: 1000u16, + prover_instance_wait_timeout_in_secs: 1000u16, + prover_instance_poll_time_in_milli_secs: 250u16, + prometheus_listener_port: 3316, + prometheus_pushgateway_url: "http://127.0.0.1:9091".to_string(), + prometheus_push_interval_ms: Some(100), + specialized_group_id: 1, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + FRI_WITNESS_VECTOR_GENERATOR_MAX_PROVER_RESERVATION_DURATION_IN_SECS=1000 + FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_WAIT_TIMEOUT_IN_SECS=1000 + FRI_WITNESS_VECTOR_GENERATOR_PROVER_INSTANCE_POLL_TIME_IN_MILLI_SECS=250 + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_LISTENER_PORT=3316 + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSHGATEWAY_URL="http://127.0.0.1:9091" + FRI_WITNESS_VECTOR_GENERATOR_PROMETHEUS_PUSH_INTERVAL_MS=100 + FRI_WITNESS_VECTOR_GENERATOR_SPECIALIZED_GROUP_ID=1 + "#; + lock.set_env(config); + + let actual = FriWitnessVectorGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/house_keeper.rs b/core/lib/env_config/src/house_keeper.rs new file mode 100644 index 000000000000..d6b2906a7a35 --- /dev/null +++ b/core/lib/env_config/src/house_keeper.rs @@ -0,0 +1,61 @@ +use zksync_config::configs::house_keeper::HouseKeeperConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for HouseKeeperConfig { + fn from_env() -> anyhow::Result { + envy_load("house_keeper", "HOUSE_KEEPER_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> HouseKeeperConfig { + HouseKeeperConfig { + l1_batch_metrics_reporting_interval_ms: 10_000, + blob_cleaning_interval_ms: 60_000, + gpu_prover_queue_reporting_interval_ms: 10_000, + prover_job_retrying_interval_ms: 300_000, + prover_stats_reporting_interval_ms: 5_000, + witness_job_moving_interval_ms: 30_000, + witness_generator_stats_reporting_interval_ms: 10_000, + fri_witness_job_moving_interval_ms: 40_000, + fri_prover_job_retrying_interval_ms: 30_000, + fri_witness_generator_job_retrying_interval_ms: 30_000, + prover_db_pool_size: 2, + fri_prover_stats_reporting_interval_ms: 30_000, + fri_proof_compressor_job_retrying_interval_ms: 30_000, + fri_proof_compressor_stats_reporting_interval_ms: 30_000, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + HOUSE_KEEPER_L1_BATCH_METRICS_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_BLOB_CLEANING_INTERVAL_MS="60000" + HOUSE_KEEPER_GPU_PROVER_QUEUE_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="300000" + HOUSE_KEEPER_PROVER_STATS_REPORTING_INTERVAL_MS="5000" + HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000" + HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000" + HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" + HOUSE_KEEPER_PROVER_DB_POOL_SIZE="2" + HOUSE_KEEPER_FRI_PROVER_STATS_REPORTING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_STATS_REPORTING_INTERVAL_MS="30000" + HOUSE_KEEPER_FRI_PROOF_COMPRESSOR_JOB_RETRYING_INTERVAL_MS="30000" + "#; + lock.set_env(config); + + let actual = HouseKeeperConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/lib.rs b/core/lib/env_config/src/lib.rs new file mode 100644 index 000000000000..a4a4af3f1ec0 --- /dev/null +++ b/core/lib/env_config/src/lib.rs @@ -0,0 +1,42 @@ +use anyhow::Context as _; +use serde::de::DeserializeOwned; + +mod alerts; +mod api; +mod chain; +mod circuit_synthesizer; +mod contract_verifier; +mod contracts; +mod database; +mod eth_client; +mod eth_sender; +mod eth_watch; +mod fetcher; +mod fri_proof_compressor; +mod fri_prover; +mod fri_prover_gateway; +mod fri_prover_group; +mod fri_witness_generator; +mod fri_witness_vector_generator; +mod house_keeper; +pub mod object_store; +mod proof_data_handler; +mod prover; +mod prover_group; +mod utils; +mod witness_generator; + +#[cfg(test)] +mod test_utils; + +pub trait FromEnv: Sized { + fn from_env() -> anyhow::Result; +} + +/// Convenience function that loads the structure from the environment variable given the prefix. +/// Panics if the config cannot be loaded from the environment variables. +pub(crate) fn envy_load(name: &str, prefix: &str) -> anyhow::Result { + envy::prefixed(prefix) + .from_env() + .with_context(|| format!("Cannot load config <{name}>")) +} diff --git a/core/lib/env_config/src/object_store.rs b/core/lib/env_config/src/object_store.rs new file mode 100644 index 000000000000..3b4afe86b522 --- /dev/null +++ b/core/lib/env_config/src/object_store.rs @@ -0,0 +1,96 @@ +use zksync_config::ObjectStoreConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ObjectStoreConfig { + fn from_env() -> anyhow::Result { + envy_load("object_store", "OBJECT_STORE_") + } +} + +/// Wrapper for `ObjectStoreConfig` that allows loading object store config using `PUBLIC_` prefix. +#[derive(Debug)] +pub struct PublicObjectStoreConfig(pub ObjectStoreConfig); + +impl FromEnv for PublicObjectStoreConfig { + fn from_env() -> anyhow::Result { + let config = envy_load("public_object_store", "PUBLIC_OBJECT_STORE_")?; + Ok(Self(config)) + } +} + +/// Wrapper for `ObjectStoreConfig` that allows loading object store config using `PROVER_` prefix. +#[derive(Debug)] +pub struct ProverObjectStoreConfig(pub ObjectStoreConfig); + +impl FromEnv for ProverObjectStoreConfig { + fn from_env() -> anyhow::Result { + let config = envy_load("prover_object_store", "PROVER_OBJECT_STORE_")?; + Ok(Self(config)) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::{configs::object_store::ObjectStoreMode, ObjectStoreConfig}; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config(bucket_base_url: &str) -> ObjectStoreConfig { + ObjectStoreConfig { + bucket_base_url: bucket_base_url.to_string(), + mode: ObjectStoreMode::FileBacked, + file_backed_base_path: "artifacts".to_string(), + gcs_credential_file_path: "/path/to/credentials.json".to_string(), + max_retries: 5, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + OBJECT_STORE_BUCKET_BASE_URL="/base/url" + OBJECT_STORE_MODE="FileBacked" + OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = ObjectStoreConfig::from_env().unwrap(); + assert_eq!(actual, expected_config("/base/url")); + } + + #[test] + fn public_bucket_config_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + PUBLIC_OBJECT_STORE_BUCKET_BASE_URL="/public_base_url" + PUBLIC_OBJECT_STORE_MODE="FileBacked" + PUBLIC_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + PUBLIC_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + PUBLIC_OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = PublicObjectStoreConfig::from_env().unwrap().0; + assert_eq!(actual, expected_config("/public_base_url")); + } + + #[test] + fn prover_bucket_config_from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + PROVER_OBJECT_STORE_BUCKET_BASE_URL="/prover_base_url" + PROVER_OBJECT_STORE_MODE="FileBacked" + PROVER_OBJECT_STORE_FILE_BACKED_BASE_PATH="artifacts" + PROVER_OBJECT_STORE_GCS_CREDENTIAL_FILE_PATH="/path/to/credentials.json" + PROVER_OBJECT_STORE_MAX_RETRIES="5" + "#; + lock.set_env(config); + let actual = ProverObjectStoreConfig::from_env().unwrap().0; + assert_eq!(actual, expected_config("/prover_base_url")); + } +} diff --git a/core/lib/env_config/src/proof_data_handler.rs b/core/lib/env_config/src/proof_data_handler.rs new file mode 100644 index 000000000000..67aad80fbe7f --- /dev/null +++ b/core/lib/env_config/src/proof_data_handler.rs @@ -0,0 +1,42 @@ +use zksync_config::configs::ProofDataHandlerConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProofDataHandlerConfig { + fn from_env() -> anyhow::Result { + envy_load("proof_data_handler", "PROOF_DATA_HANDLER_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::proof_data_handler::ProtocolVersionLoadingMode; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProofDataHandlerConfig { + ProofDataHandlerConfig { + http_port: 3320, + proof_generation_timeout_in_secs: 18000, + protocol_version_loading_mode: ProtocolVersionLoadingMode::FromEnvVar, + fri_protocol_version_id: 2, + } + } + + #[test] + fn from_env() { + let config = r#" + PROOF_DATA_HANDLER_PROOF_GENERATION_TIMEOUT_IN_SECS="18000" + PROOF_DATA_HANDLER_HTTP_PORT="3320" + PROOF_DATA_HANDLER_PROTOCOL_VERSION_LOADING_MODE="FromEnvVar" + PROOF_DATA_HANDLER_FRI_PROTOCOL_VERSION_ID="2" + "#; + let mut lock = MUTEX.lock(); + lock.set_env(config); + let actual = ProofDataHandlerConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/prover.rs b/core/lib/env_config/src/prover.rs new file mode 100644 index 000000000000..700f0fffb96a --- /dev/null +++ b/core/lib/env_config/src/prover.rs @@ -0,0 +1,197 @@ +use zksync_config::ProverConfigs; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProverConfigs { + fn from_env() -> anyhow::Result { + Ok(Self { + non_gpu: envy_load("non_gpu", "PROVER_NON_GPU_")?, + two_gpu_forty_gb_mem: envy_load( + "two_gpu_forty_gb_mem", + "PROVER_TWO_GPU_FORTY_GB_MEM_", + )?, + one_gpu_eighty_gb_mem: envy_load( + "one_gpu_eighty_gb_mem", + "PROVER_ONE_GPU_EIGHTY_GB_MEM_", + )?, + two_gpu_eighty_gb_mem: envy_load( + "two_gpu_eighty_gb_mem", + "PROVER_TWO_GPU_EIGHTY_GB_MEM_", + )?, + four_gpu_eighty_gb_mem: envy_load( + "four_gpu_eighty_gb_mem", + "PROVER_FOUR_GPU_EIGHTY_GB_MEM_", + )?, + }) + } +} + +#[cfg(test)] +mod tests { + use zksync_config::ProverConfig; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProverConfigs { + ProverConfigs { + non_gpu: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 2, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 0, + number_of_setup_slots: 2, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + two_gpu_forty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 2, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 1, + number_of_setup_slots: 5, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + one_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 4, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 2, + number_of_setup_slots: 5, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + two_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 9, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 3, + number_of_setup_slots: 9, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + four_gpu_eighty_gb_mem: ProverConfig { + prometheus_port: 3313, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + generation_timeout_in_secs: 2700u16, + number_of_threads: 18, + max_attempts: 4, + polling_duration_in_millis: 5, + setup_keys_path: "/usr/src/setup-keys".to_string(), + specialized_prover_group_id: 4, + number_of_setup_slots: 18, + assembly_receiver_port: 17791, + assembly_receiver_poll_time_in_millis: 250, + assembly_queue_capacity: 5, + }, + } + } + + const CONFIG: &str = r#" + PROVER_NON_GPU_PROMETHEUS_PORT="3313" + PROVER_NON_GPU_INITIAL_SETUP_KEY_PATH="key" + PROVER_NON_GPU_KEY_DOWNLOAD_URL="value" + PROVER_NON_GPU_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_NON_GPU_NUMBER_OF_THREADS="2" + PROVER_NON_GPU_MAX_ATTEMPTS="4" + PROVER_NON_GPU_POLLING_DURATION_IN_MILLIS=5 + PROVER_NON_GPU_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_NON_GPU_NUMBER_OF_SETUP_SLOTS=2 + PROVER_NON_GPU_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_NON_GPU_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_NON_GPU_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_NON_GPU_SPECIALIZED_PROVER_GROUP_ID=0 + + PROVER_TWO_GPU_FORTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_TWO_GPU_FORTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_TWO_GPU_FORTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_TWO_GPU_FORTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_THREADS="2" + PROVER_TWO_GPU_FORTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_TWO_GPU_FORTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_TWO_GPU_FORTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_TWO_GPU_FORTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_TWO_GPU_FORTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_TWO_GPU_FORTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=1 + + PROVER_ONE_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_ONE_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_ONE_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_ONE_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="4" + PROVER_ONE_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_ONE_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_ONE_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_ONE_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_ONE_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=2 + + PROVER_TWO_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_TWO_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_TWO_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_TWO_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="9" + PROVER_TWO_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_TWO_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_TWO_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_TWO_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=9 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_TWO_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_TWO_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=3 + + PROVER_FOUR_GPU_EIGHTY_GB_MEM_PROMETHEUS_PORT="3313" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_INITIAL_SETUP_KEY_PATH="key" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_KEY_DOWNLOAD_URL="value" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_GENERATION_TIMEOUT_IN_SECS=2700 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_THREADS="18" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_MAX_ATTEMPTS="4" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_POLLING_DURATION_IN_MILLIS=5 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_SETUP_KEYS_PATH="/usr/src/setup-keys" + PROVER_FOUR_GPU_EIGHTY_GB_MEM_NUMBER_OF_SETUP_SLOTS=18 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_PORT=17791 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_RECEIVER_POLL_TIME_IN_MILLIS=250 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_ASSEMBLY_QUEUE_CAPACITY=5 + PROVER_FOUR_GPU_EIGHTY_GB_MEM_SPECIALIZED_PROVER_GROUP_ID=4 + "#; + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + lock.set_env(CONFIG); + let actual = ProverConfigs::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/env_config/src/prover_group.rs b/core/lib/env_config/src/prover_group.rs new file mode 100644 index 000000000000..bdac82cbb9cc --- /dev/null +++ b/core/lib/env_config/src/prover_group.rs @@ -0,0 +1,149 @@ +use zksync_config::configs::ProverGroupConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for ProverGroupConfig { + fn from_env() -> anyhow::Result { + envy_load("prover_group", "PROVER_GROUP_") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> ProverGroupConfig { + ProverGroupConfig { + group_0_circuit_ids: vec![0, 18], + group_1_circuit_ids: vec![1, 4], + group_2_circuit_ids: vec![2, 5], + group_3_circuit_ids: vec![6, 7], + group_4_circuit_ids: vec![8, 9], + group_5_circuit_ids: vec![10, 11], + group_6_circuit_ids: vec![12, 13], + group_7_circuit_ids: vec![14, 15], + group_8_circuit_ids: vec![16, 17], + group_9_circuit_ids: vec![3], + region_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location".to_string(), + region_override: Some("us-central-1".to_string()), + zone_read_url: "http://metadata.google.internal/computeMetadata/v1/instance/zone".to_string(), + zone_override: Some("us-central-1-b".to_string()), + synthesizer_per_gpu: 10, + } + } + + const CONFIG: &str = r#" + PROVER_GROUP_GROUP_0_CIRCUIT_IDS="0,18" + PROVER_GROUP_GROUP_1_CIRCUIT_IDS="1,4" + PROVER_GROUP_GROUP_2_CIRCUIT_IDS="2,5" + PROVER_GROUP_GROUP_3_CIRCUIT_IDS="6,7" + PROVER_GROUP_GROUP_4_CIRCUIT_IDS="8,9" + PROVER_GROUP_GROUP_5_CIRCUIT_IDS="10,11" + PROVER_GROUP_GROUP_6_CIRCUIT_IDS="12,13" + PROVER_GROUP_GROUP_7_CIRCUIT_IDS="14,15" + PROVER_GROUP_GROUP_8_CIRCUIT_IDS="16,17" + PROVER_GROUP_GROUP_9_CIRCUIT_IDS="3" + PROVER_GROUP_REGION_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/attributes/cluster-location" + PROVER_GROUP_REGION_OVERRIDE="us-central-1" + PROVER_GROUP_ZONE_READ_URL="http://metadata.google.internal/computeMetadata/v1/instance/zone" + PROVER_GROUP_ZONE_OVERRIDE="us-central-1-b" + PROVER_GROUP_SYNTHESIZER_PER_GPU="10" + "#; + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + lock.set_env(CONFIG); + let actual = ProverGroupConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } + + #[test] + fn get_group_id_for_circuit_id() { + let prover_group_config = expected_config(); + + assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(0)); + assert_eq!(Some(0), prover_group_config.get_group_id_for_circuit_id(18)); + + assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(1)); + assert_eq!(Some(1), prover_group_config.get_group_id_for_circuit_id(4)); + + assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(2)); + assert_eq!(Some(2), prover_group_config.get_group_id_for_circuit_id(5)); + + assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(6)); + assert_eq!(Some(3), prover_group_config.get_group_id_for_circuit_id(7)); + + assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(8)); + assert_eq!(Some(4), prover_group_config.get_group_id_for_circuit_id(9)); + + assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(10)); + assert_eq!(Some(5), prover_group_config.get_group_id_for_circuit_id(11)); + + assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(12)); + assert_eq!(Some(6), prover_group_config.get_group_id_for_circuit_id(13)); + + assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(14)); + assert_eq!(Some(7), prover_group_config.get_group_id_for_circuit_id(15)); + + assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(16)); + assert_eq!(Some(8), prover_group_config.get_group_id_for_circuit_id(17)); + + assert_eq!(Some(9), prover_group_config.get_group_id_for_circuit_id(3)); + assert!(prover_group_config + .get_group_id_for_circuit_id(19) + .is_none()); + } + + #[test] + fn get_circuit_ids_for_group_id() { + let prover_group_config = expected_config(); + + assert_eq!( + Some(vec![0, 18]), + prover_group_config.get_circuit_ids_for_group_id(0) + ); + assert_eq!( + Some(vec![1, 4]), + prover_group_config.get_circuit_ids_for_group_id(1) + ); + assert_eq!( + Some(vec![2, 5]), + prover_group_config.get_circuit_ids_for_group_id(2) + ); + assert_eq!( + Some(vec![6, 7]), + prover_group_config.get_circuit_ids_for_group_id(3) + ); + assert_eq!( + Some(vec![8, 9]), + prover_group_config.get_circuit_ids_for_group_id(4) + ); + assert_eq!( + Some(vec![10, 11]), + prover_group_config.get_circuit_ids_for_group_id(5) + ); + assert_eq!( + Some(vec![12, 13]), + prover_group_config.get_circuit_ids_for_group_id(6) + ); + assert_eq!( + Some(vec![14, 15]), + prover_group_config.get_circuit_ids_for_group_id(7) + ); + assert_eq!( + Some(vec![16, 17]), + prover_group_config.get_circuit_ids_for_group_id(8) + ); + assert_eq!( + Some(vec![3]), + prover_group_config.get_circuit_ids_for_group_id(9) + ); + assert!(prover_group_config + .get_circuit_ids_for_group_id(10) + .is_none()); + } +} diff --git a/core/lib/config/src/configs/test_utils.rs b/core/lib/env_config/src/test_utils.rs similarity index 100% rename from core/lib/config/src/configs/test_utils.rs rename to core/lib/env_config/src/test_utils.rs diff --git a/core/lib/env_config/src/utils.rs b/core/lib/env_config/src/utils.rs new file mode 100644 index 000000000000..655d3b2e6d56 --- /dev/null +++ b/core/lib/env_config/src/utils.rs @@ -0,0 +1,8 @@ +use crate::{envy_load, FromEnv}; +use zksync_config::configs::PrometheusConfig; + +impl FromEnv for PrometheusConfig { + fn from_env() -> anyhow::Result { + envy_load("prometheus", "API_PROMETHEUS_") + } +} diff --git a/core/lib/env_config/src/witness_generator.rs b/core/lib/env_config/src/witness_generator.rs new file mode 100644 index 000000000000..28d7a9bbbd9d --- /dev/null +++ b/core/lib/env_config/src/witness_generator.rs @@ -0,0 +1,50 @@ +use zksync_config::configs::WitnessGeneratorConfig; + +use crate::{envy_load, FromEnv}; + +impl FromEnv for WitnessGeneratorConfig { + fn from_env() -> anyhow::Result { + envy_load("witness", "WITNESS_") + } +} + +#[cfg(test)] +mod tests { + use zksync_config::configs::witness_generator::BasicWitnessGeneratorDataSource; + + use super::*; + use crate::test_utils::EnvMutex; + + static MUTEX: EnvMutex = EnvMutex::new(); + + fn expected_config() -> WitnessGeneratorConfig { + WitnessGeneratorConfig { + generation_timeout_in_secs: 900_u16, + initial_setup_key_path: "key".to_owned(), + key_download_url: "value".to_owned(), + max_attempts: 4, + blocks_proving_percentage: Some(30), + dump_arguments_for_blocks: vec![2, 3], + last_l1_batch_to_process: None, + data_source: BasicWitnessGeneratorDataSource::FromBlob, + } + } + + #[test] + fn from_env() { + let mut lock = MUTEX.lock(); + let config = r#" + WITNESS_GENERATION_TIMEOUT_IN_SECS=900 + WITNESS_INITIAL_SETUP_KEY_PATH="key" + WITNESS_KEY_DOWNLOAD_URL="value" + WITNESS_MAX_ATTEMPTS=4 + WITNESS_DUMP_ARGUMENTS_FOR_BLOCKS="2,3" + WITNESS_BLOCKS_PROVING_PERCENTAGE="30" + WITNESS_DATA_SOURCE="FromBlob" + "#; + lock.set_env(config); + + let actual = WitnessGeneratorConfig::from_env().unwrap(); + assert_eq!(actual, expected_config()); + } +} diff --git a/core/lib/object_store/src/raw.rs b/core/lib/object_store/src/raw.rs index f369eacc2caf..bf318a61610f 100644 --- a/core/lib/object_store/src/raw.rs +++ b/core/lib/object_store/src/raw.rs @@ -1,11 +1,9 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{error, fmt, sync::Arc}; use crate::{file::FileBackedObjectStore, gcs::GoogleCloudStorage, mock::MockStore}; -use zksync_config::configs::object_store::ObjectStoreMode; -use zksync_config::ObjectStoreConfig; +use zksync_config::configs::object_store::{ObjectStoreConfig, ObjectStoreMode}; /// Bucket for [`ObjectStore`] in which objects can be placed. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] @@ -162,28 +160,6 @@ impl ObjectStoreFactory { } } - /// Creates an object store factory with the configuration taken from the environment. - /// - /// # Errors - /// - /// Invalid or missing configuration. - pub fn from_env() -> anyhow::Result { - Ok(Self::new( - ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?, - )) - } - - /// Creates an object store factory with the prover configuration taken from the environment. - /// - /// # Errors - /// - /// Invalid or missing configuration. - pub fn prover_from_env() -> anyhow::Result { - Ok(Self::new( - ObjectStoreConfig::prover_from_env().context("ObjectStoreConfig::prover_from_env()")?, - )) - } - /// Creates an object store factory with a mock in-memory store. /// All calls to [`Self::create_store()`] will return the same store; thus, the testing code /// can use [`ObjectStore`] methods for assertions. diff --git a/core/lib/prover_utils/src/region_fetcher.rs b/core/lib/prover_utils/src/region_fetcher.rs index c2b17f143328..22a0cedce494 100644 --- a/core/lib/prover_utils/src/region_fetcher.rs +++ b/core/lib/prover_utils/src/region_fetcher.rs @@ -6,29 +6,27 @@ use reqwest::Method; use zksync_config::configs::ProverGroupConfig; use zksync_utils::http_with_retries::send_request_with_retries; -pub async fn get_region() -> anyhow::Result { - let prover_group_config = ProverGroupConfig::from_env().context("ProverGroupConfig")?; - if let Some(region) = prover_group_config.region_override { - return Ok(region); +pub async fn get_region(prover_group_config: &ProverGroupConfig) -> anyhow::Result { + if let Some(region) = &prover_group_config.region_override { + return Ok(region.clone()); } - let url = prover_group_config.region_read_url; + let url = &prover_group_config.region_read_url; fetch_from_url(url).await.context("fetch_from_url()") } -pub async fn get_zone() -> anyhow::Result { - let prover_group_config = ProverGroupConfig::from_env().context("ProverGroupConfig")?; - if let Some(zone) = prover_group_config.zone_override { - return Ok(zone); +pub async fn get_zone(prover_group_config: &ProverGroupConfig) -> anyhow::Result { + if let Some(zone) = &prover_group_config.zone_override { + return Ok(zone.clone()); } - let url = prover_group_config.zone_read_url; + let url = &prover_group_config.zone_read_url; let data = fetch_from_url(url).await.context("fetch_from_url()")?; parse_zone(&data).context("parse_zone") } -async fn fetch_from_url(url: String) -> anyhow::Result { +async fn fetch_from_url(url: &str) -> anyhow::Result { let mut headers = HeaderMap::new(); headers.insert("Metadata-Flavor", HeaderValue::from_static("Google")); - let response = send_request_with_retries(&url, 5, Method::GET, Some(headers), None).await; + let response = send_request_with_retries(url, 5, Method::GET, Some(headers), None).await; response .map_err(|err| anyhow::anyhow!("Failed fetching response from url: {url}: {err:?}"))? .text() @@ -48,6 +46,8 @@ fn parse_zone(data: &str) -> anyhow::Result { #[cfg(test)] mod tests { + use zksync_config::configs::ProverGroupConfig; + use crate::region_fetcher::{get_region, get_zone, parse_zone}; #[test] @@ -65,11 +65,46 @@ mod tests { #[tokio::test] async fn test_get_region_with_override() { - assert_eq!("us-central-1", get_region().await.unwrap()); + let config = ProverGroupConfig { + group_0_circuit_ids: vec![], + group_1_circuit_ids: vec![], + group_2_circuit_ids: vec![], + group_3_circuit_ids: vec![], + group_4_circuit_ids: vec![], + group_5_circuit_ids: vec![], + group_6_circuit_ids: vec![], + group_7_circuit_ids: vec![], + group_8_circuit_ids: vec![], + group_9_circuit_ids: vec![], + region_override: Some("us-central-1".to_string()), + region_read_url: "".to_string(), + zone_override: Some("us-central-1-b".to_string()), + zone_read_url: "".to_string(), + synthesizer_per_gpu: 0, + }; + + assert_eq!("us-central-1", get_region(&config).await.unwrap()); } #[tokio::test] async fn test_get_zone_with_override() { - assert_eq!("us-central-1-b", get_zone().await.unwrap()); + let config = ProverGroupConfig { + group_0_circuit_ids: vec![], + group_1_circuit_ids: vec![], + group_2_circuit_ids: vec![], + group_3_circuit_ids: vec![], + group_4_circuit_ids: vec![], + group_5_circuit_ids: vec![], + group_6_circuit_ids: vec![], + group_7_circuit_ids: vec![], + group_8_circuit_ids: vec![], + group_9_circuit_ids: vec![], + region_override: Some("us-central-1".to_string()), + region_read_url: "".to_string(), + zone_override: Some("us-central-1-b".to_string()), + zone_read_url: "".to_string(), + synthesizer_per_gpu: 0, + }; + assert_eq!("us-central-1-b", get_zone(&config).await.unwrap()); } } diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index fd9c2cad99af..b994fbff8824 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -235,8 +235,6 @@ pub struct TxSenderConfig { pub fair_l2_gas_price: u64, pub vm_execution_cache_misses_limit: Option, pub validation_computational_gas_limit: u32, - pub default_aa: H256, - pub bootloader: H256, pub chain_id: L2ChainId, } @@ -255,8 +253,6 @@ impl TxSenderConfig { vm_execution_cache_misses_limit: web3_json_config.vm_execution_cache_misses_limit, validation_computational_gas_limit: state_keeper_config .validation_computational_gas_limit, - default_aa: state_keeper_config.default_aa_hash, - bootloader: state_keeper_config.bootloader_hash, chain_id, } } diff --git a/core/lib/zksync_core/src/api_server/web3/tests.rs b/core/lib/zksync_core/src/api_server/web3/tests.rs index 5fde6cc3aebf..0d595830d34f 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests.rs @@ -70,9 +70,9 @@ pub(crate) async fn spawn_http_server( pool: ConnectionPool, stop_receiver: watch::Receiver, ) -> ApiServerHandles { - let contracts_config = ContractsConfig::from_env().unwrap(); - let web3_config = Web3JsonRpcConfig::from_env().unwrap(); - let state_keeper_config = StateKeeperConfig::from_env().unwrap(); + let contracts_config = ContractsConfig::for_tests(); + let web3_config = Web3JsonRpcConfig::for_tests(); + let state_keeper_config = StateKeeperConfig::for_tests(); let api_config = InternalApiConfig::new(network_config, &web3_config, &contracts_config); let tx_sender_config = TxSenderConfig::new(&state_keeper_config, &web3_config, api_config.l2_chain_id); @@ -103,7 +103,7 @@ pub(crate) async fn spawn_http_server( #[tokio::test] async fn http_server_can_start() { let pool = ConnectionPool::test_pool().await; - let network_config = NetworkConfig::from_env().unwrap(); + let network_config = NetworkConfig::for_tests(); let mut storage = pool.access_storage().await.unwrap(); if storage.blocks_dal().is_genesis_needed().await.unwrap() { ensure_genesis_state( diff --git a/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs b/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs index 0052b091f2a2..1d022e4700d4 100644 --- a/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs +++ b/core/lib/zksync_core/src/data_fetchers/token_list/one_inch.rs @@ -55,19 +55,3 @@ impl FetcherImpl for OneInchTokenListFetcher { pub(super) struct OneInchTokensResponse { pub tokens: HashMap, } - -#[tokio::test] -#[ignore] -// We can't rely on 1inch API in unit tests, so we ignore this test. -async fn test_fetch_one_inch_token_list() { - let mut config = FetcherConfig::from_env().unwrap(); - config.token_list.url = "https://api.1inch.exchange".to_string(); - - let fetcher = OneInchTokenListFetcher::new(&config); - - let token_list = fetcher - .fetch_token_list() - .await - .expect("failed get token list"); - assert!(!token_list.is_empty(), "Token list is empty"); -} diff --git a/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs b/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs index 3b7125f8c632..a046c23ea2d2 100644 --- a/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs +++ b/core/lib/zksync_core/src/data_fetchers/token_price/coingecko.rs @@ -165,32 +165,3 @@ pub struct CoinGeckoTokenPrice { #[serde(with = "UnsignedRatioSerializeAsDecimal")] pub usd: Ratio, } - -#[tokio::test] -#[ignore] -async fn test_fetch_coingecko_prices() { - let mut config = FetcherConfig::from_env().unwrap(); - config.token_price.url = "https://api.coingecko.com".to_string(); - - let fetcher = CoinGeckoFetcher::new(&config); - - let tokens = vec![ - ETHEREUM_ADDRESS, - Address::from_str("6b175474e89094c44da98b954eedeac495271d0f").expect("DAI"), - Address::from_str("1f9840a85d5af5bf1d1762f925bdaddc4201f984").expect("UNI"), - Address::from_str("514910771af9ca656af840dff83e8264ecf986ca").expect("LINK"), - ]; - - let token_prices = fetcher - .fetch_token_price(&tokens) - .await - .expect("failed get tokens price"); - assert_eq!( - token_prices.len(), - tokens.len(), - "not all data was received" - ); - for token_address in tokens { - assert!(token_prices.get(&token_address).is_some()); - } -} diff --git a/core/lib/zksync_core/src/eth_sender/tests.rs b/core/lib/zksync_core/src/eth_sender/tests.rs index b2c1fa77e608..08069df6414c 100644 --- a/core/lib/zksync_core/src/eth_sender/tests.rs +++ b/core/lib/zksync_core/src/eth_sender/tests.rs @@ -65,8 +65,8 @@ impl EthSenderTester { history: Vec, non_ordering_confirmations: bool, ) -> Self { - let eth_sender_config = ETHSenderConfig::from_env().unwrap(); - let contracts_config = ContractsConfig::from_env().unwrap(); + let eth_sender_config = ETHSenderConfig::for_tests(); + let contracts_config = ContractsConfig::for_tests(); let aggregator_config = SenderConfig { aggregated_proof_sizes: vec![1], ..eth_sender_config.sender.clone() @@ -100,7 +100,7 @@ impl EthSenderTester { .await .unwrap(), ); - let store_factory = ObjectStoreFactory::from_env().unwrap(); + let store_factory = ObjectStoreFactory::mock(); let aggregator = EthTxAggregator::new( SenderConfig { diff --git a/core/lib/zksync_core/src/eth_watch/mod.rs b/core/lib/zksync_core/src/eth_watch/mod.rs index 640f96707e4a..fdb629bce28a 100644 --- a/core/lib/zksync_core/src/eth_watch/mod.rs +++ b/core/lib/zksync_core/src/eth_watch/mod.rs @@ -4,7 +4,6 @@ //! Poll interval is configured using the `ETH_POLL_INTERVAL` constant. //! Number of confirmations is configured using the `CONFIRMATIONS_FOR_ETH_EVENT` environment variable. -use anyhow::Context as _; use tokio::{sync::watch, task::JoinHandle}; use std::time::Duration; @@ -188,18 +187,18 @@ impl EthWatch { } pub async fn start_eth_watch( + config: ETHWatchConfig, pool: ConnectionPool, eth_gateway: E, diamond_proxy_addr: Address, governance: (Contract, Address), stop_receiver: watch::Receiver, ) -> anyhow::Result>> { - let eth_watch = ETHWatchConfig::from_env().context("ETHWatchConfig::from_env()")?; let eth_client = EthHttpQueryClient::new( eth_gateway, diamond_proxy_addr, Some(governance.1), - eth_watch.confirmations_for_eth_event, + config.confirmations_for_eth_event, ); let mut eth_watch = EthWatch::new( @@ -207,7 +206,7 @@ pub async fn start_eth_watch( Some(governance.0), eth_client, &pool, - eth_watch.poll_interval(), + config.poll_interval(), ) .await; diff --git a/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs b/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs index dcd209860205..5b41ee74ac9a 100644 --- a/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs +++ b/core/lib/zksync_core/src/house_keeper/prover_queue_monitor.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use zksync_config::configs::ProverGroupConfig; use zksync_dal::ConnectionPool; @@ -10,13 +9,19 @@ use zksync_prover_utils::periodic_job::PeriodicJob; pub struct ProverStatsReporter { reporting_interval_ms: u64, prover_connection_pool: ConnectionPool, + config: ProverGroupConfig, } impl ProverStatsReporter { - pub fn new(reporting_interval_ms: u64, prover_connection_pool: ConnectionPool) -> Self { + pub fn new( + reporting_interval_ms: u64, + prover_connection_pool: ConnectionPool, + config: ProverGroupConfig, + ) -> Self { Self { reporting_interval_ms, prover_connection_pool, + config, } } } @@ -28,13 +33,12 @@ impl PeriodicJob for ProverStatsReporter { const SERVICE_NAME: &'static str = "ProverStatsReporter"; async fn run_routine_task(&mut self) -> anyhow::Result<()> { - let prover_group_config = - ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; let mut conn = self.prover_connection_pool.access_storage().await.unwrap(); let stats = conn.prover_dal().get_prover_jobs_stats_per_circuit().await; for (circuit_name, stats) in stats.into_iter() { - let group_id = prover_group_config + let group_id = self + .config .get_group_id_for_circuit_id(circuit_name_to_numeric_index(&circuit_name).unwrap()) .unwrap(); diff --git a/core/lib/zksync_core/src/l1_gas_price/singleton.rs b/core/lib/zksync_core/src/l1_gas_price/singleton.rs index 766dc246a949..4808dee548ba 100644 --- a/core/lib/zksync_core/src/l1_gas_price/singleton.rs +++ b/core/lib/zksync_core/src/l1_gas_price/singleton.rs @@ -3,13 +3,17 @@ use anyhow::Context as _; use std::sync::Arc; use tokio::sync::{watch, OnceCell}; use tokio::task::JoinHandle; -use zksync_config::{ETHClientConfig, GasAdjusterConfig}; +use zksync_config::GasAdjusterConfig; use zksync_eth_client::clients::http::QueryClient; /// Special struct for creating a singleton of `GasAdjuster`. -/// This is needed only for running the server. This struct uses all configs from env. -#[derive(Debug, Default)] -pub struct GasAdjusterSingleton(OnceCell>, Error>>); +/// This is needed only for running the server. +#[derive(Debug)] +pub struct GasAdjusterSingleton { + web3_url: String, + gas_adjuster_config: GasAdjusterConfig, + singleton: OnceCell>, Error>>, +} #[derive(thiserror::Error, Debug, Clone)] #[error(transparent)] @@ -22,21 +26,21 @@ impl From for Error { } impl GasAdjusterSingleton { - pub fn new() -> Self { - Default::default() + pub fn new(web3_url: String, gas_adjuster_config: GasAdjusterConfig) -> Self { + Self { + web3_url, + gas_adjuster_config, + singleton: OnceCell::new(), + } } pub async fn get_or_init(&mut self) -> Result>, Error> { let adjuster = self - .0 + .singleton .get_or_init(|| async { - let eth_client_config = - ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?; let query_client = - QueryClient::new(ð_client_config.web3_url).context("QueryClient::new()")?; - let gas_adjuster_config = - GasAdjusterConfig::from_env().context("GasAdjusterConfig::from_env()")?; - let adjuster = GasAdjuster::new(query_client.clone(), gas_adjuster_config) + QueryClient::new(&self.web3_url).context("QueryClient::new()")?; + let adjuster = GasAdjuster::new(query_client.clone(), self.gas_adjuster_config) .await .context("GasAdjuster::new()")?; Ok(Arc::new(adjuster)) @@ -48,10 +52,9 @@ impl GasAdjusterSingleton { pub async fn get_or_init_bounded( &mut self, ) -> anyhow::Result>>> { - let config = GasAdjusterConfig::from_env().context("GasAdjusterConfig::from_env()")?; let adjuster = self.get_or_init().await.context("get_or_init()")?; Ok(Arc::new(BoundedGasAdjuster::new( - config.max_l1_gas_price(), + self.gas_adjuster_config.max_l1_gas_price(), adjuster, ))) } @@ -60,7 +63,7 @@ impl GasAdjusterSingleton { self, stop_signal: watch::Receiver, ) -> Option>> { - let gas_adjuster = self.0.get()?.clone(); + let gas_adjuster = self.singleton.get()?.clone(); Some(tokio::spawn( async move { gas_adjuster?.run(stop_signal).await }, )) diff --git a/core/lib/zksync_core/src/lib.rs b/core/lib/zksync_core/src/lib.rs index 2e35cf573a94..9294a3e65def 100644 --- a/core/lib/zksync_core/src/lib.rs +++ b/core/lib/zksync_core/src/lib.rs @@ -5,28 +5,24 @@ use std::{net::Ipv4Addr, str::FromStr, sync::Arc, time::Instant}; use anyhow::Context as _; use futures::channel::oneshot; use prometheus_exporter::PrometheusExporterConfig; +use temp_config_store::TempConfigStore; use tokio::{sync::watch, task::JoinHandle}; use zksync_circuit_breaker::{ l1_txs::FailedL1TransactionChecker, replication_lag::ReplicationLagChecker, CircuitBreaker, CircuitBreakerChecker, CircuitBreakerError, }; -use zksync_config::configs::api::MerkleTreeApiConfig; -use zksync_config::configs::contracts::ProverAtGenesis; -use zksync_config::configs::{ - api::{HealthCheckConfig, Web3JsonRpcConfig}, - chain::{ - self, CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, - StateKeeperConfig, - }, - database::MerkleTreeMode, - house_keeper::HouseKeeperConfig, - FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, - ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, -}; use zksync_config::{ - ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, FetcherConfig, - ProverConfigs, + configs::{ + api::{MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + contracts::ProverAtGenesis, + database::MerkleTreeMode, + }, + ApiConfig, ContractsConfig, DBConfig, ETHSenderConfig, }; use zksync_contracts::{governance_contract, BaseSystemContracts}; use zksync_dal::{ @@ -36,7 +32,7 @@ use zksync_eth_client::clients::http::QueryClient; use zksync_eth_client::EthInterface; use zksync_eth_client::{clients::http::PKSigningClient, BoundEthInterface}; use zksync_health_check::{CheckHealth, HealthStatus, ReactiveHealthCheck}; -use zksync_object_store::ObjectStoreFactory; +use zksync_object_store::{ObjectStore, ObjectStoreFactory}; use zksync_prover_utils::periodic_job::PeriodicJob; use zksync_queued_job_processor::JobProcessor; use zksync_state::PostgresStorageCaches; @@ -65,6 +61,7 @@ pub mod proof_data_handler; pub mod reorg_detector; pub mod state_keeper; pub mod sync_layer; +pub mod temp_config_store; pub mod witness_generator; use crate::api_server::healthcheck::HealthCheckHandle; @@ -327,6 +324,7 @@ impl FromStr for Components { } pub async fn initialize_components( + configs: &TempConfigStore, components: Vec, use_prometheus_push_gateway: bool, ) -> anyhow::Result<( @@ -337,7 +335,10 @@ pub async fn initialize_components( )> { tracing::info!("Starting the components: {components:?}"); - let db_config = DBConfig::from_env().context("DbConfig::from_env()")?; + let db_config = configs + .db_config + .clone() + .ok_or_else(|| anyhow::anyhow!("db_config"))?; let connection_pool = ConnectionPool::builder(DbVariant::Master) .build() .await @@ -353,10 +354,18 @@ pub async fn initialize_components( .context("failed to build replica_connection_pool")?; let mut healthchecks: Vec> = Vec::new(); - let contracts_config = ContractsConfig::from_env().context("ContractsConfig::from_env()")?; - let eth_client_config = ETHClientConfig::from_env().context("ETHClientConfig::from_env()")?; - let circuit_breaker_config = - CircuitBreakerConfig::from_env().context("CircuitBreakerConfig::from_env()")?; + let contracts_config = configs + .contracts_config + .clone() + .ok_or_else(|| anyhow::anyhow!("contracts_config"))?; + let eth_client_config = configs + .eth_client_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_client_config"))?; + let circuit_breaker_config = configs + .circuit_breaker_config + .clone() + .ok_or_else(|| anyhow::anyhow!("circuit_breaker_config"))?; let circuit_breaker_checker = CircuitBreakerChecker::new( circuit_breakers_for_components(&components, &circuit_breaker_config) @@ -369,13 +378,20 @@ pub async fn initialize_components( }); let query_client = QueryClient::new(ð_client_config.web3_url).unwrap(); - let mut gas_adjuster = GasAdjusterSingleton::new(); + let gas_adjuster_config = configs + .gas_adjuster_config + .ok_or_else(|| anyhow::anyhow!("gas_adjuster_config"))?; + let mut gas_adjuster = + GasAdjusterSingleton::new(eth_client_config.web3_url.clone(), gas_adjuster_config); let (stop_sender, stop_receiver) = watch::channel(false); let (cb_sender, cb_receiver) = oneshot::channel(); // Prometheus exporter and circuit breaker checker should run for every component configuration. - let prom_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?; + let prom_config = configs + .prometheus_config + .clone() + .ok_or_else(|| anyhow::anyhow!("prometheus_config"))?; let prom_config = if use_prometheus_push_gateway { PrometheusExporterConfig::push(prom_config.gateway_endpoint(), prom_config.push_interval()) } else { @@ -403,10 +419,18 @@ pub async fn initialize_components( || components.contains(&Component::ContractVerificationApi) || components.contains(&Component::ApiTranslator) { - let api_config = ApiConfig::from_env().context("ApiConfig::from_env()")?; - let state_keeper_config = - StateKeeperConfig::from_env().context("StateKeeperConfig::from_env()")?; - let network_config = NetworkConfig::from_env().context("NetworkConfig::from_env()")?; + let api_config = configs + .api_config + .clone() + .ok_or_else(|| anyhow::anyhow!("api_config"))?; + let state_keeper_config = configs + .state_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("state_keeper_config"))?; + let network_config = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; let tx_sender_config = TxSenderConfig::new( &state_keeper_config, &api_config.web3_json_rpc, @@ -426,7 +450,7 @@ pub async fn initialize_components( if components.contains(&Component::HttpApi) { storage_caches = Some( - build_storage_caches(&replica_connection_pool, &mut task_futures) + build_storage_caches(configs, &replica_connection_pool, &mut task_futures) .context("build_storage_caches()")?, ); @@ -465,7 +489,7 @@ pub async fn initialize_components( if components.contains(&Component::WsApi) { let storage_caches = match storage_caches { Some(storage_caches) => storage_caches, - None => build_storage_caches(&replica_connection_pool, &mut task_futures) + None => build_storage_caches(configs, &replica_connection_pool, &mut task_futures) .context("build_storage_caches()")?, }; @@ -515,6 +539,12 @@ pub async fn initialize_components( } } + let object_store_config = configs + .object_store_config + .clone() + .ok_or_else(|| anyhow::anyhow!("object_store_config"))?; + let store_factory = ObjectStoreFactory::new(object_store_config); + if components.contains(&Component::StateKeeper) { let started_at = Instant::now(); tracing::info!("initializing State Keeper"); @@ -525,11 +555,21 @@ pub async fn initialize_components( add_state_keeper_to_task_futures( &mut task_futures, &contracts_config, - StateKeeperConfig::from_env().context("StateKeeperConfig::from_env()")?, - &NetworkConfig::from_env().context("NetworkConfig::from_env()")?, + configs + .state_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("state_keeper_config"))?, + &configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?, &db_config, - &MempoolConfig::from_env().context("MempoolConfig::from_env()")?, + &configs + .mempool_config + .clone() + .ok_or_else(|| anyhow::anyhow!("mempool_config"))?, bounded_gas_adjuster, + store_factory.create_store().await, stop_receiver.clone(), ) .await @@ -549,8 +589,13 @@ pub async fn initialize_components( .await .context("failed to build eth_watch_pool")?; let governance = (governance_contract(), contracts_config.governance_addr); + let eth_watch_config = configs + .eth_watch_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_watch_config"))?; task_futures.push( start_eth_watch( + eth_watch_config, eth_watch_pool, query_client.clone(), main_zksync_contract_address, @@ -565,8 +610,6 @@ pub async fn initialize_components( tracing::info!("initialized ETH-Watcher in {elapsed:?}"); } - let store_factory = ObjectStoreFactory::from_env().context("ObjectStoreFactor::from_env()")?; - if components.contains(&Component::EthTxAggregator) { let started_at = Instant::now(); tracing::info!("initializing ETH-TxAggregator"); @@ -579,7 +622,10 @@ pub async fn initialize_components( .await .context("failed to build eth_sender_prover_pool")?; - let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?; + let eth_sender = configs + .eth_sender_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_sender_config"))?; let eth_client = PKSigningClient::from_config(ð_sender, &contracts_config, ð_client_config); let nonce = eth_client.pending_nonce("eth_sender").await.unwrap(); @@ -612,7 +658,10 @@ pub async fn initialize_components( .build() .await .context("failed to build eth_manager_pool")?; - let eth_sender = ETHSenderConfig::from_env().context("ETHSenderConfig::from_env()")?; + let eth_sender = configs + .eth_sender_config + .clone() + .ok_or_else(|| anyhow::anyhow!("eth_sender_config"))?; let eth_client = PKSigningClient::from_config(ð_sender, &contracts_config, ð_client_config); let eth_tx_manager_actor = EthTxManager::new( @@ -633,8 +682,14 @@ pub async fn initialize_components( if components.contains(&Component::DataFetcher) { let started_at = Instant::now(); - let fetcher_config = FetcherConfig::from_env().context("FetcherConfig::from_env()")?; - let eth_network = chain::NetworkConfig::from_env().context("NetworkConfig::from_env()")?; + let fetcher_config = configs + .fetcher_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fetcher_config"))?; + let eth_network = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; tracing::info!("initializing data fetchers"); task_futures.extend(run_data_fetchers( &fetcher_config, @@ -648,6 +703,7 @@ pub async fn initialize_components( } add_trees_to_task_futures( + configs, &mut task_futures, &mut healthchecks, &components, @@ -657,6 +713,7 @@ pub async fn initialize_components( .await .context("add_trees_to_task_futures()")?; add_witness_generator_to_task_futures( + configs, &mut task_futures, &components, &connection_pool, @@ -672,13 +729,15 @@ pub async fn initialize_components( .build() .await .context("failed to build singleton connection_pool")?; + let network_config = configs + .network_config + .clone() + .ok_or_else(|| anyhow::anyhow!("network_config"))?; add_basic_witness_input_producer_to_task_futures( &mut task_futures, &singleton_connection_pool, &store_factory, - NetworkConfig::from_env() - .context("NetworkConfig::from_env()")? - .zksync_network_id, + network_config.zksync_network_id, stop_receiver.clone(), ) .await @@ -686,14 +745,21 @@ pub async fn initialize_components( } if components.contains(&Component::Housekeeper) { - add_house_keeper_to_task_futures(&mut task_futures, &store_factory) + add_house_keeper_to_task_futures(configs, &mut task_futures, &store_factory) .await .context("add_house_keeper_to_task_futures()")?; } if components.contains(&Component::ProofDataHandler) { task_futures.push(tokio::spawn(proof_data_handler::run_server( - ProofDataHandlerConfig::from_env().context("ProofDataHandlerConfig::from_env()")?, + configs + .proof_data_handler_config + .clone() + .ok_or_else(|| anyhow::anyhow!("proof_data_handler_config"))?, + configs + .contracts_config + .clone() + .ok_or_else(|| anyhow::anyhow!("contracts_config"))?, store_factory.create_store().await, connection_pool.clone(), stop_receiver.clone(), @@ -705,8 +771,10 @@ pub async fn initialize_components( replica_connection_pool, ))); - let healtcheck_api_config = - HealthCheckConfig::from_env().context("HealthCheckConfig::from_env()")?; + let healtcheck_api_config = configs + .health_check_config + .clone() + .ok_or_else(|| anyhow::anyhow!("health_check_config"))?; let health_check_handle = HealthCheckHandle::spawn_server(healtcheck_api_config.bind_addr(), healthchecks); @@ -725,6 +793,7 @@ async fn add_state_keeper_to_task_futures, + object_store: Box, stop_receiver: watch::Receiver, ) -> anyhow::Result<()> { let fair_l2_gas_price = state_keeper_config.fair_l2_gas_price; @@ -763,6 +832,7 @@ async fn add_state_keeper_to_task_futures>>, healthchecks: &mut Vec>, components: &[Component], @@ -795,11 +866,18 @@ async fn add_trees_to_task_futures( anyhow::bail!("Tree backup mode is disabled"); } - let db_config = DBConfig::from_env().context("DBConfig::from_env()")?; - let operation_config = - OperationsManagerConfig::from_env().context("OperationManagerConfig::from_env()")?; - let api_config = ApiConfig::from_env() - .context("ApiConfig::from_env()")? + let db_config = configs + .db_config + .clone() + .ok_or_else(|| anyhow::anyhow!("db_config"))?; + let operation_config = configs + .operations_manager_config + .clone() + .ok_or_else(|| anyhow::anyhow!("operations_manager_config"))?; + let api_config = configs + .api_config + .clone() + .ok_or_else(|| anyhow::anyhow!("api_config"))? .merkle_tree; let api_config = components .contains(&Component::TreeApi) @@ -859,7 +937,8 @@ async fn run_tree( }; tracing::info!("Initializing Merkle tree in {mode_str} mode"); - let config = MetadataCalculatorConfig::for_main_node(db_config, operation_manager, mode); + let config = + MetadataCalculatorConfig::for_main_node(&db_config.merkle_tree, operation_manager, mode); let metadata_calculator = MetadataCalculator::new(&config).await; if let Some(api_config) = api_config { let address = (Ipv4Addr::UNSPECIFIED, api_config.port).into(); @@ -915,6 +994,7 @@ async fn add_basic_witness_input_producer_to_task_futures( } async fn add_witness_generator_to_task_futures( + configs: &TempConfigStore, task_futures: &mut Vec>>, components: &[Component], connection_pool: &ConnectionPool, @@ -949,8 +1029,10 @@ async fn add_witness_generator_to_task_futures( .protocol_versions_dal() .protocol_version_for(&vk_commitments) .await; - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; + let config = configs + .witness_generator_config + .clone() + .ok_or_else(|| anyhow::anyhow!("witness_generator_config"))?; let task = match component_type { AggregationRound::BasicCircuits => { let witness_generator = BasicWitnessGenerator::new( @@ -1007,11 +1089,14 @@ async fn add_witness_generator_to_task_futures( } async fn add_house_keeper_to_task_futures( + configs: &TempConfigStore, task_futures: &mut Vec>>, store_factory: &ObjectStoreFactory, ) -> anyhow::Result<()> { - let house_keeper_config = - HouseKeeperConfig::from_env().context("HouseKeeperConfig::from_env()")?; + let house_keeper_config = configs + .house_keeper_config + .clone() + .ok_or_else(|| anyhow::anyhow!("house_keeper_config"))?; let connection_pool = ConnectionPool::singleton(DbVariant::Replica) .build() .await @@ -1026,16 +1111,20 @@ async fn add_house_keeper_to_task_futures( .build() .await .context("failed to build a prover_connection_pool")?; + let prover_group_config = configs + .prover_group_config + .clone() + .ok_or_else(|| anyhow::anyhow!("prover_group_config"))?; + let prover_configs = configs + .prover_configs + .clone() + .ok_or_else(|| anyhow::anyhow!("prover_configs"))?; let gpu_prover_queue = GpuProverQueueMonitor::new( - ProverGroupConfig::from_env() - .context("ProverGroupConfig::from_env()")? - .synthesizer_per_gpu, + prover_group_config.synthesizer_per_gpu, house_keeper_config.gpu_prover_queue_reporting_interval_ms, prover_connection_pool.clone(), ); - let config = ProverConfigs::from_env() - .context("ProverCOnfigs::from_env()")? - .non_gpu; + let config = prover_configs.non_gpu.clone(); let prover_job_retry_manager = ProverJobRetryManager::new( config.max_attempts, config.proof_generation_timeout(), @@ -1045,6 +1134,7 @@ async fn add_house_keeper_to_task_futures( let prover_stats_reporter = ProverStatsReporter::new( house_keeper_config.prover_stats_reporting_interval_ms, prover_connection_pool.clone(), + prover_group_config.clone(), ); let waiting_to_queued_witness_job_mover = WaitingToQueuedWitnessJobMover::new( house_keeper_config.witness_job_moving_interval_ms, @@ -1070,7 +1160,10 @@ async fn add_house_keeper_to_task_futures( task_futures.push(tokio::spawn(prover_job_retry_manager.run())); // All FRI Prover related components are configured below. - let fri_prover_config = FriProverConfig::from_env().context("FriProverConfig::from_env()")?; + let fri_prover_config = configs + .fri_prover_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_prover_config"))?; let fri_prover_job_retry_manager = FriProverJobRetryManager::new( fri_prover_config.max_attempts, fri_prover_config.proof_generation_timeout(), @@ -1079,8 +1172,10 @@ async fn add_house_keeper_to_task_futures( ); task_futures.push(tokio::spawn(fri_prover_job_retry_manager.run())); - let fri_witness_gen_config = - FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env")?; + let fri_witness_gen_config = configs + .fri_witness_generator_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_witness_generator_config"))?; let fri_witness_gen_job_retry_manager = FriWitnessGeneratorJobRetryManager::new( fri_witness_gen_config.max_attempts, fri_witness_gen_config.witness_generation_timeout(), @@ -1113,8 +1208,10 @@ async fn add_house_keeper_to_task_futures( ); task_futures.push(tokio::spawn(fri_prover_stats_reporter.run())); - let proof_compressor_config = - FriProofCompressorConfig::from_env().context("FriProofCompressorConfig")?; + let proof_compressor_config = configs + .fri_proof_compressor_config + .clone() + .ok_or_else(|| anyhow::anyhow!("fri_proof_compressor_config"))?; let fri_proof_compressor_stats_reporter = FriProofCompressorStatsReporter::new( house_keeper_config.fri_proof_compressor_stats_reporting_interval_ms, prover_connection_pool.clone(), @@ -1132,10 +1229,14 @@ async fn add_house_keeper_to_task_futures( } fn build_storage_caches( + configs: &TempConfigStore, replica_connection_pool: &ConnectionPool, task_futures: &mut Vec>>, ) -> anyhow::Result { - let rpc_config = Web3JsonRpcConfig::from_env().context("Web3JsonRpcConfig::from_env()")?; + let rpc_config = configs + .web3_json_rpc_config + .clone() + .ok_or_else(|| anyhow::anyhow!("web3_json_rpc_config"))?; let factory_deps_capacity = rpc_config.factory_deps_cache_size() as u64; let initial_writes_capacity = rpc_config.initial_writes_cache_size() as u64; let values_capacity = rpc_config.latest_values_cache_size() as u64; diff --git a/core/lib/zksync_core/src/metadata_calculator/mod.rs b/core/lib/zksync_core/src/metadata_calculator/mod.rs index 3602a79fa829..0f50b2bb09b7 100644 --- a/core/lib/zksync_core/src/metadata_calculator/mod.rs +++ b/core/lib/zksync_core/src/metadata_calculator/mod.rs @@ -7,7 +7,7 @@ use std::time::Duration; use zksync_config::configs::{ chain::OperationsManagerConfig, - database::{DBConfig, MerkleTreeMode}, + database::{MerkleTreeConfig, MerkleTreeMode}, }; use zksync_dal::{ConnectionPool, StorageProcessor}; use zksync_health_check::{HealthUpdater, ReactiveHealthCheck}; @@ -81,19 +81,19 @@ pub struct MetadataCalculatorConfig<'a> { impl<'a> MetadataCalculatorConfig<'a> { pub(crate) fn for_main_node( - db_config: &'a DBConfig, + merkle_tree_config: &'a MerkleTreeConfig, operation_config: &'a OperationsManagerConfig, mode: MetadataCalculatorModeConfig<'a>, ) -> Self { Self { - db_path: &db_config.merkle_tree.path, + db_path: &merkle_tree_config.path, mode, delay_interval: operation_config.delay_interval(), - max_l1_batches_per_iter: db_config.merkle_tree.max_l1_batches_per_iter, - multi_get_chunk_size: db_config.merkle_tree.multi_get_chunk_size, - block_cache_capacity: db_config.merkle_tree.block_cache_size(), - memtable_capacity: db_config.merkle_tree.memtable_capacity(), - stalled_writes_timeout: db_config.merkle_tree.stalled_writes_timeout(), + max_l1_batches_per_iter: merkle_tree_config.max_l1_batches_per_iter, + multi_get_chunk_size: merkle_tree_config.multi_get_chunk_size, + block_cache_capacity: merkle_tree_config.block_cache_size(), + memtable_capacity: merkle_tree_config.memtable_capacity(), + stalled_writes_timeout: merkle_tree_config.stalled_writes_timeout(), } } } diff --git a/core/lib/zksync_core/src/metadata_calculator/tests.rs b/core/lib/zksync_core/src/metadata_calculator/tests.rs index dca85e1f7d28..dd72410f950d 100644 --- a/core/lib/zksync_core/src/metadata_calculator/tests.rs +++ b/core/lib/zksync_core/src/metadata_calculator/tests.rs @@ -5,7 +5,7 @@ use tokio::sync::{mpsc, watch}; use std::{future::Future, ops, panic, path::Path, time::Duration}; -use zksync_config::{configs::chain::OperationsManagerConfig, DBConfig}; +use zksync_config::configs::{chain::OperationsManagerConfig, database::MerkleTreeConfig}; use zksync_contracts::BaseSystemContracts; use zksync_dal::{ConnectionPool, StorageProcessor}; use zksync_health_check::{CheckHealth, HealthStatus}; @@ -243,11 +243,11 @@ async fn shutting_down_calculator() { let pool = ConnectionPool::test_pool().await; let prover_pool = ConnectionPool::test_pool().await; let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); - let (db_config, mut operation_config) = create_config(temp_dir.path()); + let (merkle_tree_config, mut operation_config) = create_config(temp_dir.path()); operation_config.delay_interval = 30_000; // ms; chosen to be larger than `RUN_TIMEOUT` let calculator = setup_calculator_with_options( - &db_config, + &merkle_tree_config, &operation_config, &pool, MetadataCalculatorModeConfig::Lightweight, @@ -371,12 +371,12 @@ pub(crate) async fn setup_calculator( pool: &ConnectionPool, ) -> (MetadataCalculator, Box) { let store_factory = &ObjectStoreFactory::mock(); - let (db_config, operation_manager) = create_config(db_path); + let (merkle_tree_config, operation_manager) = create_config(db_path); let mode = MetadataCalculatorModeConfig::Full { store_factory: Some(store_factory), }; let calculator = - setup_calculator_with_options(&db_config, &operation_manager, pool, mode).await; + setup_calculator_with_options(&merkle_tree_config, &operation_manager, pool, mode).await; (calculator, store_factory.create_store().await) } @@ -386,10 +386,11 @@ async fn setup_lightweight_calculator(db_path: &Path, pool: &ConnectionPool) -> setup_calculator_with_options(&db_config, &operation_config, pool, mode).await } -fn create_config(db_path: &Path) -> (DBConfig, OperationsManagerConfig) { - let mut db_config = DBConfig::from_env().unwrap(); - db_config.merkle_tree.path = path_to_string(&db_path.join("new")); - db_config.backup_interval_ms = 0; +fn create_config(db_path: &Path) -> (MerkleTreeConfig, OperationsManagerConfig) { + let db_config = MerkleTreeConfig { + path: path_to_string(&db_path.join("new")), + ..Default::default() + }; let operation_config = OperationsManagerConfig { delay_interval: 50, // ms @@ -398,13 +399,13 @@ fn create_config(db_path: &Path) -> (DBConfig, OperationsManagerConfig) { } async fn setup_calculator_with_options( - db_config: &DBConfig, + merkle_tree_config: &MerkleTreeConfig, operation_config: &OperationsManagerConfig, pool: &ConnectionPool, mode: MetadataCalculatorModeConfig<'_>, ) -> MetadataCalculator { let calculator_config = - MetadataCalculatorConfig::for_main_node(db_config, operation_config, mode); + MetadataCalculatorConfig::for_main_node(merkle_tree_config, operation_config, mode); let metadata_calculator = MetadataCalculator::new(&calculator_config).await; let mut storage = pool.access_storage().await.unwrap(); diff --git a/core/lib/zksync_core/src/proof_data_handler/mod.rs b/core/lib/zksync_core/src/proof_data_handler/mod.rs index 2c3454662324..898ac4652ba2 100644 --- a/core/lib/zksync_core/src/proof_data_handler/mod.rs +++ b/core/lib/zksync_core/src/proof_data_handler/mod.rs @@ -18,21 +18,21 @@ use zksync_types::{ mod request_processor; -fn fri_l1_verifier_config_from_env() -> anyhow::Result { - let config = ContractsConfig::from_env().context("ContractsConfig::from_env()")?; - Ok(L1VerifierConfig { +fn fri_l1_verifier_config(contracts_config: &ContractsConfig) -> L1VerifierConfig { + L1VerifierConfig { params: VerifierParams { - recursion_node_level_vk_hash: config.fri_recursion_node_level_vk_hash, - recursion_leaf_level_vk_hash: config.fri_recursion_leaf_level_vk_hash, + recursion_node_level_vk_hash: contracts_config.fri_recursion_node_level_vk_hash, + recursion_leaf_level_vk_hash: contracts_config.fri_recursion_leaf_level_vk_hash, // The base layer commitment is not used in the FRI prover verification. recursion_circuits_set_vks_hash: H256::zero(), }, - recursion_scheduler_level_vk_hash: config.snark_wrapper_vk_hash, - }) + recursion_scheduler_level_vk_hash: contracts_config.snark_wrapper_vk_hash, + } } pub(crate) async fn run_server( config: ProofDataHandlerConfig, + contracts_config: ContractsConfig, blob_store: Box, pool: ConnectionPool, mut stop_receiver: watch::Receiver, @@ -41,9 +41,7 @@ pub(crate) async fn run_server( tracing::debug!("Starting proof data handler server on {bind_address}"); let l1_verifier_config: Option = match config.protocol_version_loading_mode { ProtocolVersionLoadingMode::FromDb => None, - ProtocolVersionLoadingMode::FromEnvVar => { - Some(fri_l1_verifier_config_from_env().context("fri_l1_verified_config_from_env()")?) - } + ProtocolVersionLoadingMode::FromEnvVar => Some(fri_l1_verifier_config(&contracts_config)), }; let get_proof_gen_processor = RequestProcessor::new(blob_store, pool, config, l1_verifier_config); diff --git a/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs b/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs index 84ee799fe12f..c3f03fd86321 100644 --- a/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs +++ b/core/lib/zksync_core/src/state_keeper/batch_executor/tests/tester.rs @@ -6,7 +6,6 @@ use tempfile::TempDir; use multivm::interface::{L1BatchEnv, SystemEnv}; use multivm::vm_latest::constants::INITIAL_STORAGE_WRITE_PUBDATA_BYTES; -use zksync_config::configs::chain::StateKeeperConfig; use zksync_contracts::{get_loadnext_contract, test_contracts::LoadnextContractExecutionParams}; use zksync_dal::ConnectionPool; use zksync_state::RocksdbStorage; @@ -41,14 +40,17 @@ pub(super) struct TestConfig { impl TestConfig { pub(super) fn new() -> Self { - // It's OK to use env config here, since we would load the postgres URL from there anyway. - let config = StateKeeperConfig::from_env().unwrap(); + // Values are taken from the actual env config used in local development. + // These values remain static for a long time, so it's a better alternative than to introduce dependency + // on env config. + const MAX_ALLOWED_TX_GAS_LIMIT: u32 = 4000000000; + const VALIDATION_COMPUTATIONAL_GAS_LIMIT: u32 = 300000; Self { vm_gas_limit: None, save_call_traces: false, - max_allowed_tx_gas_limit: config.max_allowed_l2_tx_gas_limit, - validation_computational_gas_limit: config.validation_computational_gas_limit, + max_allowed_tx_gas_limit: MAX_ALLOWED_TX_GAS_LIMIT, + validation_computational_gas_limit: VALIDATION_COMPUTATIONAL_GAS_LIMIT, upload_witness_inputs_to_gcs: false, } } diff --git a/core/lib/zksync_core/src/state_keeper/io/mempool.rs b/core/lib/zksync_core/src/state_keeper/io/mempool.rs index fced87479283..f10ad87580c0 100644 --- a/core/lib/zksync_core/src/state_keeper/io/mempool.rs +++ b/core/lib/zksync_core/src/state_keeper/io/mempool.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{ @@ -14,7 +13,7 @@ use multivm::vm_latest::utils::fee::derive_base_fee_and_gas_per_pubdata; use zksync_config::configs::chain::StateKeeperConfig; use zksync_dal::ConnectionPool; use zksync_mempool::L2TxFilter; -use zksync_object_store::ObjectStoreFactory; +use zksync_object_store::ObjectStore; use zksync_types::{ block::MiniblockHeader, protocol_version::ProtocolUpgradeTx, witness_block_state::WitnessBlockState, Address, L1BatchNumber, L2ChainId, MiniblockNumber, @@ -47,6 +46,7 @@ use crate::{ pub(crate) struct MempoolIO { mempool: MempoolGuard, pool: ConnectionPool, + object_store: Box, timeout_sealer: TimeoutSealer, filter: L2TxFilter, current_miniblock_number: MiniblockNumber, @@ -297,11 +297,8 @@ where self.miniblock_sealer_handle.wait_for_all_commands().await; if let Some(witness_witness_block_state) = witness_block_state { - let object_store = ObjectStoreFactory::from_env() - .context("ObjectsStoreFactor::from_env()")? - .create_store() - .await; - match object_store + match self + .object_store .put(self.current_l1_batch_number(), &witness_witness_block_state) .await { @@ -405,6 +402,7 @@ impl MempoolIO { #[allow(clippy::too_many_arguments)] pub(in crate::state_keeper) async fn new( mempool: MempoolGuard, + object_store: Box, miniblock_sealer_handle: MiniblockSealerHandle, l1_gas_price_provider: Arc, pool: ConnectionPool, @@ -439,6 +437,7 @@ impl MempoolIO { Self { mempool, + object_store, pool, timeout_sealer: TimeoutSealer::new(config), filter: L2TxFilter::default(), diff --git a/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs b/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs index d685e1fc81d9..875bf89e0481 100644 --- a/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs +++ b/core/lib/zksync_core/src/state_keeper/io/tests/tester.rs @@ -2,6 +2,7 @@ use multivm::vm_latest::constants::BLOCK_GAS_LIMIT; use std::{sync::Arc, time::Duration}; +use zksync_object_store::ObjectStoreFactory; use zksync_config::configs::chain::StateKeeperConfig; use zksync_config::GasAdjusterConfig; @@ -72,18 +73,17 @@ impl Tester { MiniblockSealer::new(pool.clone(), miniblock_sealer_capacity); tokio::spawn(miniblock_sealer.run()); - let base_contract_hashes = self.base_system_contracts.hashes(); let config = StateKeeperConfig { fair_l2_gas_price: self.fair_l2_gas_price(), - bootloader_hash: base_contract_hashes.bootloader, - default_aa_hash: base_contract_hashes.default_aa, virtual_blocks_interval: 1, virtual_blocks_per_miniblock: 1, ..StateKeeperConfig::default() }; + let object_store = ObjectStoreFactory::mock().create_store().await; let l2_erc20_bridge_addr = Address::repeat_byte(0x5a); // Isn't relevant. let io = MempoolIO::new( mempool.clone(), + object_store, miniblock_sealer_handle, gas_adjuster, pool, diff --git a/core/lib/zksync_core/src/state_keeper/mod.rs b/core/lib/zksync_core/src/state_keeper/mod.rs index 18401001b4de..bdc1f90e2066 100644 --- a/core/lib/zksync_core/src/state_keeper/mod.rs +++ b/core/lib/zksync_core/src/state_keeper/mod.rs @@ -1,4 +1,5 @@ use tokio::sync::watch; +use zksync_object_store::ObjectStore; use std::sync::Arc; @@ -44,6 +45,7 @@ pub(crate) async fn create_state_keeper( mempool: MempoolGuard, l1_gas_price_provider: Arc, miniblock_sealer_handle: MiniblockSealerHandle, + object_store: Box, stop_receiver: watch::Receiver, ) -> ZkSyncStateKeeper where @@ -67,6 +69,7 @@ where let io = MempoolIO::new( mempool, + object_store, miniblock_sealer_handle, l1_gas_price_provider, pool, diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs index 4683d9cf8798..1472e05fad90 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/gas.rs @@ -56,7 +56,14 @@ mod tests { #[test] fn test_gas_seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + max_single_tx_gas: 6000000, + reject_tx_at_gas_percentage: 0.95, + close_block_at_gas_percentage: 0.95, + ..Default::default() + }; + let criterion = GasCriterion; // Empty block should fit into gas criterion. diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs index 143104d5ae53..61f30d724a70 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/pubdata_bytes.rs @@ -58,7 +58,13 @@ mod tests { #[test] fn seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + reject_tx_at_eth_params_percentage: 0.95, + close_block_at_eth_params_percentage: 0.95, + ..Default::default() + }; + let criterion = PubDataBytesCriterion; let block_execution_metrics = ExecutionMetrics { diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs index b434c3ba42a7..4c21c41e5e40 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/slots.rs @@ -36,7 +36,12 @@ mod tests { #[test] fn test_slots_seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + transaction_slots: 2, + ..Default::default() + }; + let criterion = SlotsCriterion; let almost_full_block_resolution = criterion.should_seal( diff --git a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs index 4672ac36e557..ed24e3719338 100644 --- a/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs +++ b/core/lib/zksync_core/src/state_keeper/seal_criteria/criteria/tx_encoding_size.rs @@ -47,7 +47,13 @@ mod tests { #[test] fn seal_criterion() { - let config = StateKeeperConfig::from_env().unwrap(); + // Create an empty config and only setup fields relevant for the test. + let config = StateKeeperConfig { + reject_tx_at_geometry_percentage: 0.95, + close_block_at_geometry_percentage: 0.95, + ..Default::default() + }; + let criterion = TxEncodingSizeCriterion; let empty_block_resolution = criterion.should_seal( diff --git a/core/lib/zksync_core/src/sync_layer/tests.rs b/core/lib/zksync_core/src/sync_layer/tests.rs index 8fe61fe16e6d..e454c0098e12 100644 --- a/core/lib/zksync_core/src/sync_layer/tests.rs +++ b/core/lib/zksync_core/src/sync_layer/tests.rs @@ -578,7 +578,7 @@ async fn fetcher_with_real_server() { let mut tx_hashes = VecDeque::from(tx_hashes); // Start the API server. - let network_config = NetworkConfig::from_env().unwrap(); + let network_config = NetworkConfig::for_tests(); let (stop_sender, stop_receiver) = watch::channel(false); let server_handles = spawn_http_server(&network_config, pool.clone(), stop_receiver.clone()).await; diff --git a/core/lib/zksync_core/src/temp_config_store.rs b/core/lib/zksync_core/src/temp_config_store.rs new file mode 100644 index 000000000000..618f8ac6a274 --- /dev/null +++ b/core/lib/zksync_core/src/temp_config_store.rs @@ -0,0 +1,47 @@ +use zksync_config::{ + configs::{ + api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig}, + chain::{ + CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig, + StateKeeperConfig, + }, + house_keeper::HouseKeeperConfig, + FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig, + ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig, + }, + ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig, + FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs, +}; + +// TODO (QIT-22): This structure is going to be removed when components will be respnsible for their own configs. +/// A temporary config store allowing to pass deserialized configs from `zksync_server` to `zksync_core`. +/// All the configs are optional, since for some component combination it is not needed to pass all the configs. +#[derive(Debug)] +pub struct TempConfigStore { + pub health_check_config: Option, + pub merkle_tree_api_config: Option, + pub web3_json_rpc_config: Option, + pub circuit_breaker_config: Option, + pub mempool_config: Option, + pub network_config: Option, + pub operations_manager_config: Option, + pub state_keeper_config: Option, + pub house_keeper_config: Option, + pub fri_proof_compressor_config: Option, + pub fri_prover_config: Option, + pub fri_witness_generator_config: Option, + pub prometheus_config: Option, + pub proof_data_handler_config: Option, + pub prover_group_config: Option, + pub witness_generator_config: Option, + pub api_config: Option, + pub contracts_config: Option, + pub db_config: Option, + pub eth_client_config: Option, + pub eth_sender_config: Option, + pub eth_watch_config: Option, + pub fetcher_config: Option, + pub gas_adjuster_config: Option, + pub prover_configs: Option, + pub object_store_config: Option, +} diff --git a/core/lib/zksync_core/src/witness_generator/basic_circuits.rs b/core/lib/zksync_core/src/witness_generator/basic_circuits.rs index e48d1f1db6d9..de6ede7e14fd 100644 --- a/core/lib/zksync_core/src/witness_generator/basic_circuits.rs +++ b/core/lib/zksync_core/src/witness_generator/basic_circuits.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use rand::Rng; use serde::{Deserialize, Serialize}; @@ -13,8 +12,9 @@ use std::{ use multivm::vm_latest::{ constants::MAX_CYCLES_FOR_TX, HistoryDisabled, SimpleMemory, StorageOracle as VmStorageOracle, }; -use zksync_config::configs::witness_generator::BasicWitnessGeneratorDataSource; -use zksync_config::configs::WitnessGeneratorConfig; +use zksync_config::configs::{ + witness_generator::BasicWitnessGeneratorDataSource, WitnessGeneratorConfig, +}; use zksync_dal::ConnectionPool; use zksync_object_store::{Bucket, ObjectStore, ObjectStoreFactory, StoredObject}; use zksync_queued_job_processor::JobProcessor; @@ -88,14 +88,13 @@ impl BasicWitnessGenerator { } async fn process_job_impl( + config: WitnessGeneratorConfig, object_store: Arc, connection_pool: ConnectionPool, prover_connection_pool: ConnectionPool, basic_job: BasicWitnessGeneratorJob, started_at: Instant, ) -> anyhow::Result> { - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; let BasicWitnessGeneratorJob { block_number, job } = basic_job; if let Some(blocks_proving_percentage) = config.blocks_proving_percentage { @@ -213,7 +212,9 @@ impl JobProcessor for BasicWitnessGenerator { started_at: Instant, ) -> tokio::task::JoinHandle>> { let object_store = Arc::clone(&self.object_store); + let config = self.config.clone(); tokio::spawn(Self::process_job_impl( + config, object_store, self.connection_pool.clone(), self.prover_connection_pool.clone(), diff --git a/core/lib/zksync_core/src/witness_generator/node_aggregation.rs b/core/lib/zksync_core/src/witness_generator/node_aggregation.rs index 7ba3b746065d..349ab56b1556 100644 --- a/core/lib/zksync_core/src/witness_generator/node_aggregation.rs +++ b/core/lib/zksync_core/src/witness_generator/node_aggregation.rs @@ -1,4 +1,3 @@ -use anyhow::Context as _; use async_trait::async_trait; use std::{collections::HashMap, env, time::Instant}; @@ -76,11 +75,10 @@ impl NodeAggregationWitnessGenerator { } fn process_job_sync( + config: WitnessGeneratorConfig, node_job: NodeAggregationWitnessGeneratorJob, started_at: Instant, ) -> anyhow::Result { - let config = - WitnessGeneratorConfig::from_env().context("WitnessGeneratorConfig::from_env()")?; let NodeAggregationWitnessGeneratorJob { block_number, job } = node_job; tracing::info!( @@ -162,7 +160,8 @@ impl JobProcessor for NodeAggregationWitnessGenerator { job: NodeAggregationWitnessGeneratorJob, started_at: Instant, ) -> tokio::task::JoinHandle> { - tokio::task::spawn_blocking(move || Self::process_job_sync(job, started_at)) + let config = self.config.clone(); + tokio::task::spawn_blocking(move || Self::process_job_sync(config, job, started_at)) } async fn save_result( diff --git a/etc/env/base/chain.toml b/etc/env/base/chain.toml index adf0b7d572c9..db9d87107c41 100644 --- a/etc/env/base/chain.toml +++ b/etc/env/base/chain.toml @@ -45,9 +45,6 @@ reject_tx_at_eth_params_percentage=0.95 # it takes more percentage of the max block gas capacity than this value. reject_tx_at_gas_percentage=0.95 -bootloader_hash="0x0100038581be3d0e201b3cc45d151ef5cc59eb3a0f146ad44f0f72abf00b594c" -default_aa_hash="0x0100038dc66b69be75ec31653c64cb931678299b9b659472772b2550b703f41c" - # The price the operator spends on 1 gas of computation in wei. fair_l2_gas_price=250000000 diff --git a/prover/circuit_synthesizer/Cargo.toml b/prover/circuit_synthesizer/Cargo.toml index 3bee4194af91..d8de49fb7655 100644 --- a/prover/circuit_synthesizer/Cargo.toml +++ b/prover/circuit_synthesizer/Cargo.toml @@ -12,6 +12,7 @@ zksync_dal = { path = "../../core/lib/dal" } zksync_types = { path = "../../core/lib/types" } zksync_queued_job_processor = { path = "../../core/lib/queued_job_processor" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_utils = { path = "../../core/lib/utils" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } diff --git a/prover/circuit_synthesizer/src/circuit_synthesizer.rs b/prover/circuit_synthesizer/src/circuit_synthesizer.rs index 6fbd9d267ab7..ce1a5251d53c 100644 --- a/prover/circuit_synthesizer/src/circuit_synthesizer.rs +++ b/prover/circuit_synthesizer/src/circuit_synthesizer.rs @@ -17,6 +17,7 @@ use zksync_config::configs::prover_group::ProverGroupConfig; use zksync_config::configs::CircuitSynthesizerConfig; use zksync_config::ProverConfigs; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::{CircuitKey, ObjectStore, ObjectStoreError, ObjectStoreFactory}; use zksync_prover_fri_utils::socket_utils::send_assembly; use zksync_prover_utils::numeric_index_to_circuit_name; @@ -88,10 +89,10 @@ impl CircuitSynthesizer { blob_store: store_factory.create_store().await, allowed_circuit_types: allowed_circuit_types .map(|x| x.into_iter().map(|x| x.1).collect()), - region: get_region() + region: get_region(&prover_groups) .await .map_err(CircuitSynthesizerError::GetRegionFailed)?, - zone: get_zone() + zone: get_zone(&prover_groups) .await .map_err(CircuitSynthesizerError::GetZoneFailed)?, vk_commitments, diff --git a/prover/circuit_synthesizer/src/main.rs b/prover/circuit_synthesizer/src/main.rs index 0462c5c78610..5c816d00903e 100644 --- a/prover/circuit_synthesizer/src/main.rs +++ b/prover/circuit_synthesizer/src/main.rs @@ -3,9 +3,12 @@ use prometheus_exporter::PrometheusExporterConfig; use structopt::StructOpt; use tokio::{sync::oneshot, sync::watch}; -use zksync_config::configs::{AlertsConfig, CircuitSynthesizerConfig, ProverGroupConfig}; +use zksync_config::configs::{ + AlertsConfig, CircuitSynthesizerConfig, ObjectStoreConfig, ProverGroupConfig, +}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStoreFactory; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -50,10 +53,12 @@ async fn main() -> anyhow::Result<()> { .context("failed to build a connection pool")?; let vk_commitments = get_cached_commitments(); + let object_store_config = + ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?; let circuit_synthesizer = CircuitSynthesizer::new( config.clone(), ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?, - &ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?, + &ObjectStoreFactory::new(object_store_config), vk_commitments, pool, ) diff --git a/prover/proof_fri_compressor/Cargo.toml b/prover/proof_fri_compressor/Cargo.toml index 02ba39e393c8..45ffe756e196 100644 --- a/prover/proof_fri_compressor/Cargo.toml +++ b/prover/proof_fri_compressor/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_utils = { path = "../../core/lib/utils" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } diff --git a/prover/proof_fri_compressor/src/main.rs b/prover/proof_fri_compressor/src/main.rs index bb3a8bf3d54c..a805bb3e3216 100644 --- a/prover/proof_fri_compressor/src/main.rs +++ b/prover/proof_fri_compressor/src/main.rs @@ -9,6 +9,7 @@ use prometheus_exporter::PrometheusExporterConfig; use zksync_config::configs::FriProofCompressorConfig; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; use zksync_queued_job_processor::JobProcessor; use zksync_utils::wait_for_tasks::wait_for_tasks; @@ -52,8 +53,9 @@ async fn main() -> anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let blob_store = ObjectStoreFactory::prover_from_env() - .context("ObjectSToreFactor::prover_from_env()")? + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let blob_store = ObjectStoreFactory::new(object_store_config.0) .create_store() .await; let proof_compressor = ProofCompressor::new( diff --git a/prover/prover/Cargo.toml b/prover/prover/Cargo.toml index 736d42e734e7..d0b66f3e3f7f 100644 --- a/prover/prover/Cargo.toml +++ b/prover/prover/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_utils = { path = "../../core/lib/utils" } zksync_prover_utils = { path = "../../core/lib/prover_utils" } zksync_circuit_breaker = { path = "../../core/lib/circuit_breaker" } diff --git a/prover/prover/src/run.rs b/prover/prover/src/run.rs index ae66c3a88d82..53a9c821f040 100644 --- a/prover/prover/src/run.rs +++ b/prover/prover/src/run.rs @@ -7,10 +7,13 @@ use queues::Buffer; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::{ - configs::{api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig}, + configs::{ + api::PrometheusConfig, prover_group::ProverGroupConfig, AlertsConfig, ObjectStoreConfig, + }, ApiConfig, ProverConfig, ProverConfigs, }; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStoreFactory; use zksync_prover_utils::region_fetcher::{get_region, get_zone}; use zksync_types::proofs::{GpuProverInstanceStatus, SocketAddress}; @@ -32,8 +35,12 @@ async fn graceful_shutdown() -> anyhow::Result> { .context("ProverConfigs")? .non_gpu .assembly_receiver_port; - let region = get_region().await.context("get_region()")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let region = get_region(&prover_group_config) + .await + .context("get_region()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let address = SocketAddress { host, port }; Ok(async move { pool.access_storage() @@ -119,8 +126,12 @@ pub async fn run() -> anyhow::Result<()> { .prometheus }; - let region = get_region().await.context("get_regtion()")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let region = get_region(&prover_group_config) + .await + .context("get_region()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let (stop_signal_sender, stop_signal_receiver) = oneshot::channel(); let mut stop_signal_sender = Some(stop_signal_sender); @@ -143,9 +154,8 @@ pub async fn run() -> anyhow::Result<()> { // Though we still need to create a channel because circuit breaker expects `stop_receiver`. let (_stop_sender, stop_receiver) = tokio::sync::watch::channel(false); - let circuit_ids = ProverGroupConfig::from_env() - .context("ProverGroupConfig::from_env()")? - .get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id); + let circuit_ids = + prover_group_config.get_circuit_ids_for_group_id(prover_config.specialized_prover_group_id); tracing::info!( "Starting proof generation for circuits: {circuit_ids:?} \ @@ -183,7 +193,9 @@ pub async fn run() -> anyhow::Result<()> { ))); let params = ProverParams::new(&prover_config); - let store_factory = ObjectStoreFactory::from_env().context("ObjectStoreFactory::from_env()")?; + let object_store_config = + ObjectStoreConfig::from_env().context("ObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config); let circuit_provider_pool = ConnectionPool::singleton(DbVariant::Prover) .build() diff --git a/prover/prover_fri/Cargo.toml b/prover/prover_fri/Cargo.toml index b351b6bf604f..314f034a9a63 100644 --- a/prover/prover_fri/Cargo.toml +++ b/prover/prover_fri/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } vlog = { path = "../../core/lib/vlog" } zksync_object_store = { path = "../../core/lib/object_store" } diff --git a/prover/prover_fri/src/gpu_prover_job_processor.rs b/prover/prover_fri/src/gpu_prover_job_processor.rs index 79fef4bcc4bc..1919d5888214 100644 --- a/prover/prover_fri/src/gpu_prover_job_processor.rs +++ b/prover/prover_fri/src/gpu_prover_job_processor.rs @@ -19,6 +19,7 @@ pub mod gpu_prover { use zksync_config::configs::fri_prover_group::FriProverGroupConfig; use zksync_config::configs::FriProverConfig; use zksync_dal::ConnectionPool; + use zksync_env_config::FromEnv; use zksync_object_store::ObjectStore; use zksync_prover_fri_types::{CircuitWrapper, FriProofWrapper, ProverServiceDataKey}; use zksync_queued_job_processor::{async_trait, JobProcessor}; @@ -285,6 +286,7 @@ pub mod gpu_prover { &config.specialized_group_id ); let prover_setup_metadata_list = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(config.specialized_group_id) .context( "At least one circuit should be configured for group when running in FromMemory mode", diff --git a/prover/prover_fri/src/main.rs b/prover/prover_fri/src/main.rs index f387393c0727..9d2a8e4ff9f1 100644 --- a/prover/prover_fri/src/main.rs +++ b/prover/prover_fri/src/main.rs @@ -7,10 +7,13 @@ use tokio::task::JoinHandle; use prometheus_exporter::PrometheusExporterConfig; use zksync_config::configs::fri_prover_group::FriProverGroupConfig; -use zksync_config::configs::FriProverConfig; -use zksync_config::ObjectStoreConfig; +use zksync_config::configs::{FriProverConfig, ProverGroupConfig}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{ + object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig}, + FromEnv, +}; use zksync_object_store::{ObjectStore, ObjectStoreFactory}; use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; @@ -33,7 +36,9 @@ async fn graceful_shutdown(port: u16) -> anyhow::Result .await .context("failed to build a connection pool")?; let host = local_ip().context("Failed obtaining local IP address")?; - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let address = SocketAddress { host, port }; Ok(async move { pool.access_storage() @@ -83,22 +88,23 @@ async fn main() -> anyhow::Result<()> { .context("Error setting Ctrl+C handler")?; let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false); - let blob_store = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let object_store_factory = ObjectStoreFactory::new(object_store_config.0); + let public_object_store_config = + PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?; let public_blob_store = match prover_config.shall_save_to_public_bucket { false => None, true => Some( - ObjectStoreFactory::new( - ObjectStoreConfig::public_from_env() - .context("ObjectStoreConfig::public_from_env()")?, - ) - .create_store() - .await, + ObjectStoreFactory::new(public_object_store_config.0) + .create_store() + .await, ), }; let specialized_group_id = prover_config.specialized_group_id; let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(specialized_group_id) .unwrap_or_default(); let circuit_ids_for_round_to_be_proven = @@ -117,7 +123,7 @@ async fn main() -> anyhow::Result<()> { let prover_tasks = get_prover_tasks( prover_config, stop_receiver.clone(), - blob_store, + object_store_factory, public_blob_store, pool, circuit_ids_for_round_to_be_proven, @@ -202,7 +208,10 @@ async fn get_prover_tasks( let witness_vector_queue = FixedSizeQueue::new(prover_config.queue_capacity); let shared_witness_vector_queue = Arc::new(Mutex::new(witness_vector_queue)); let consumer = shared_witness_vector_queue.clone(); - let zone = get_zone().await.context("get_zone()")?; + + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let local_ip = local_ip().context("Failed obtaining local IP address")?; let address = SocketAddress { host: local_ip, diff --git a/prover/prover_fri/src/prover_job_processor.rs b/prover/prover_fri/src/prover_job_processor.rs index bedee075b83d..bc53465af91f 100644 --- a/prover/prover_fri/src/prover_job_processor.rs +++ b/prover/prover_fri/src/prover_job_processor.rs @@ -22,6 +22,7 @@ use zkevm_test_harness::prover_utils::{prove_base_layer_circuit, prove_recursion use zksync_config::configs::fri_prover_group::FriProverGroupConfig; use zksync_config::configs::FriProverConfig; use zksync_dal::ConnectionPool; +use zksync_env_config::FromEnv; use zksync_object_store::ObjectStore; use zksync_prover_fri_types::{CircuitWrapper, FriProofWrapper, ProverJob, ProverServiceDataKey}; use zksync_prover_fri_utils::fetch_next_circuit; @@ -271,6 +272,7 @@ pub fn load_setup_data_cache(config: &FriProverConfig) -> anyhow::Result anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let store_factory = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactory::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config.0); let proof_submitter = PeriodicApiStruct { blob_store: store_factory.create_store().await, diff --git a/prover/setup_key_generator_and_server/Cargo.toml b/prover/setup_key_generator_and_server/Cargo.toml index 2b22a5519437..22b1cef97bd6 100644 --- a/prover/setup_key_generator_and_server/Cargo.toml +++ b/prover/setup_key_generator_and_server/Cargo.toml @@ -21,6 +21,7 @@ path = "src/main.rs" zksync_types = { path = "../../core/lib/types" } vlog = { path = "../../core/lib/vlog" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } circuit_testing = { git = "https://github.com/matter-labs/era-circuit_testing.git", branch = "main" } api = { git = "https://github.com/matter-labs/era-heavy-ops-service.git", branch = "v1.3.3", features = [ diff --git a/prover/setup_key_generator_and_server/src/lib.rs b/prover/setup_key_generator_and_server/src/lib.rs index 141a17b7d4f2..a2e4f0998f85 100644 --- a/prover/setup_key_generator_and_server/src/lib.rs +++ b/prover/setup_key_generator_and_server/src/lib.rs @@ -11,6 +11,7 @@ use zkevm_test_harness::witness::vk_set_generator::circuits_for_vk_generation; use zksync_types::circuit::GEOMETRY_CONFIG; use zksync_config::ProverConfigs; +use zksync_env_config::FromEnv; use zksync_types::circuit::{LEAF_SPLITTING_FACTOR, NODE_SPLITTING_FACTOR, SCHEDULER_UPPER_BOUND}; pub fn get_setup_for_circuit_type(circuit_type: u8) -> anyhow::Result> { let filepath = get_setup_key_file_path(circuit_type).context("get_setup_key_file_path()")?; diff --git a/prover/vk_setup_data_generator_server_fri/Cargo.toml b/prover/vk_setup_data_generator_server_fri/Cargo.toml index bae690ed0410..59df5f37cf89 100644 --- a/prover/vk_setup_data_generator_server_fri/Cargo.toml +++ b/prover/vk_setup_data_generator_server_fri/Cargo.toml @@ -32,6 +32,7 @@ circuit_definitions = { git = "https://github.com/matter-labs/era-zkevm_test_har ] } shivini = { git = "https://github.com/matter-labs/era-shivini.git", branch = "main", optional = true } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } anyhow = "1.0" tracing = "0.1" diff --git a/prover/vk_setup_data_generator_server_fri/src/lib.rs b/prover/vk_setup_data_generator_server_fri/src/lib.rs index ed391fbc86df..75125aa81508 100644 --- a/prover/vk_setup_data_generator_server_fri/src/lib.rs +++ b/prover/vk_setup_data_generator_server_fri/src/lib.rs @@ -43,6 +43,7 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use zkevm_test_harness::prover_utils::create_base_layer_setup_data; use zksync_config::configs::FriProverConfig; +use zksync_env_config::FromEnv; use zksync_types::proofs::AggregationRound; use zksync_types::zkevm_test_harness::abstract_zksync_circuit::concrete_circuits::ZkSyncCircuit; use zksync_types::zkevm_test_harness::bellman::bn256::Bn256; diff --git a/prover/witness_generator/Cargo.toml b/prover/witness_generator/Cargo.toml index ef2ce86bc579..a8dce0547b23 100644 --- a/prover/witness_generator/Cargo.toml +++ b/prover/witness_generator/Cargo.toml @@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries. [dependencies] zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_system_constants = { path = "../../core/lib/constants" } prometheus_exporter = { path = "../../core/lib/prometheus_exporter" } vlog = { path = "../../core/lib/vlog" } diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index f44afb5909d1..357aff42eb3e 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -6,8 +6,11 @@ use std::time::Instant; use structopt::StructOpt; use tokio::sync::watch; use zksync_config::configs::{FriWitnessGeneratorConfig, PrometheusConfig}; -use zksync_config::ObjectStoreConfig; use zksync_dal::{connection::DbVariant, ConnectionPool}; +use zksync_env_config::{ + object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig}, + FromEnv, +}; use zksync_object_store::ObjectStoreFactory; use zksync_prover_utils::get_stop_signal_receiver; use zksync_queued_job_processor::JobProcessor; @@ -71,8 +74,9 @@ async fn main() -> anyhow::Result<()> { let started_at = Instant::now(); let use_push_gateway = opt.batch_size.is_some(); - let store_factory = - ObjectStoreFactory::prover_from_env().context("ObjectStoreFactor::prover_from_env()")?; + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let store_factory = ObjectStoreFactory::new(object_store_config.0); let config = FriWitnessGeneratorConfig::from_env().context("FriWitnessGeneratorConfig::from_env()")?; let prometheus_config = PrometheusConfig::from_env().context("PrometheusConfig::from_env()")?; @@ -122,17 +126,16 @@ async fn main() -> anyhow::Result<()> { }; let prometheus_task = prometheus_config.run(stop_receiver.clone()); + let public_object_store_config = + PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?; let witness_generator_task = match opt.round { AggregationRound::BasicCircuits => { let public_blob_store = match config.shall_save_to_public_bucket { false => None, true => Some( - ObjectStoreFactory::new( - ObjectStoreConfig::public_from_env() - .context("ObjectStoreConfig::public_from_env()")?, - ) - .create_store() - .await, + ObjectStoreFactory::new(public_object_store_config.0) + .create_store() + .await, ), }; let generator = BasicWitnessGenerator::new( diff --git a/prover/witness_generator/tests/basic_test.rs b/prover/witness_generator/tests/basic_test.rs index 42ab161bd5e3..1c8d00ff35e7 100644 --- a/prover/witness_generator/tests/basic_test.rs +++ b/prover/witness_generator/tests/basic_test.rs @@ -2,6 +2,7 @@ use std::time::Instant; use serde::Serialize; use zksync_config::ObjectStoreConfig; +use zksync_env_config::FromEnv; use zksync_object_store::{AggregationsKey, FriCircuitKey, ObjectStoreFactory}; use zksync_types::proofs::{ AggregationRound, LeafAggregationJobMetadata, NodeAggregationJobMetadata, diff --git a/prover/witness_vector_generator/Cargo.toml b/prover/witness_vector_generator/Cargo.toml index 4360eb7516f2..e8f94849e30e 100644 --- a/prover/witness_vector_generator/Cargo.toml +++ b/prover/witness_vector_generator/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" zksync_types = { path = "../../core/lib/types" } zksync_dal = { path = "../../core/lib/dal" } zksync_config = { path = "../../core/lib/config" } +zksync_env_config = { path = "../../core/lib/env_config" } zksync_object_store = { path = "../../core/lib/object_store" } zksync_prover_fri_utils = { path = "../prover_fri_utils" } zksync_utils = { path = "../../core/lib/utils" } diff --git a/prover/witness_vector_generator/src/main.rs b/prover/witness_vector_generator/src/main.rs index 48fc372023ab..16eeb660da38 100644 --- a/prover/witness_vector_generator/src/main.rs +++ b/prover/witness_vector_generator/src/main.rs @@ -7,9 +7,10 @@ use tokio::{sync::oneshot, sync::watch}; use crate::generator::WitnessVectorGenerator; use zksync_config::configs::fri_prover_group::FriProverGroupConfig; -use zksync_config::configs::FriWitnessVectorGeneratorConfig; +use zksync_config::configs::{FriWitnessVectorGeneratorConfig, ProverGroupConfig}; use zksync_dal::connection::DbVariant; use zksync_dal::ConnectionPool; +use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; use zksync_object_store::ObjectStoreFactory; use zksync_prover_fri_utils::get_all_circuit_id_round_tuples_for; use zksync_prover_utils::region_fetcher::get_zone; @@ -58,16 +59,20 @@ async fn main() -> anyhow::Result<()> { .build() .await .context("failed to build a connection pool")?; - let blob_store = ObjectStoreFactory::prover_from_env() - .context("ObjectStoreFactor::prover_from_env()")? + let object_store_config = + ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; + let blob_store = ObjectStoreFactory::new(object_store_config.0) .create_store() .await; let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env() + .context("FriProverGroupConfig::from_env()")? .get_circuit_ids_for_group_id(specialized_group_id) .unwrap_or_default(); let circuit_ids_for_round_to_be_proven = get_all_circuit_id_round_tuples_for(circuit_ids_for_round_to_be_proven); - let zone = get_zone().await.context("get_zone()")?; + let prover_group_config = + ProverGroupConfig::from_env().context("ProverGroupConfig::from_env()")?; + let zone = get_zone(&prover_group_config).await.context("get_zone()")?; let vk_commitments = get_cached_commitments(); let witness_vector_generator = WitnessVectorGenerator::new( blob_store,