From 1004cd4fa7814e258761537b3917c193ca650dba Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 7 Jun 2024 16:24:40 +0400 Subject: [PATCH] Move object store Signed-off-by: Danil --- core/bin/zksync_server/src/main.rs | 3 ++- core/bin/zksync_server/src/node_builder.rs | 4 +--- core/lib/config/src/configs/fri_prover.rs | 1 - core/lib/config/src/configs/general.rs | 4 +++- core/lib/config/src/testonly.rs | 1 - core/lib/env_config/src/fri_prover.rs | 3 +-- core/lib/protobuf_config/src/general.rs | 5 ++++- core/lib/protobuf_config/src/proto/config/general.proto | 2 ++ core/lib/protobuf_config/src/proto/config/prover.proto | 1 - core/lib/protobuf_config/src/prover.rs | 7 ------- core/lib/zksync_core_leftovers/src/lib.rs | 3 --- .../lib/zksync_core_leftovers/src/temp_config_store/mod.rs | 4 +++- prover/config/src/lib.rs | 4 +++- 13 files changed, 19 insertions(+), 23 deletions(-) diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index cb50a1c47527..c51cc5380253 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -17,7 +17,7 @@ use zksync_config::{ ProtectiveReadsWriterConfig, Secrets, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, - GenesisConfig, PostgresConfig, SnapshotsCreatorConfig, + GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, }; use zksync_core_leftovers::{ genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler, @@ -307,5 +307,6 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + core_object_store: ObjectStoreConfig::from_env().ok(), }) } diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index e283549e14f8..904e260dcbd4 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -149,9 +149,7 @@ impl MainNodeBuilder { } fn add_object_store_layer(mut self) -> anyhow::Result { - let object_store_config = try_load_config!(self.configs.prover_config) - .core_object_store - .context("object_store_config")?; + let object_store_config = try_load_config!(self.configs.core_object_store); self.node .add_layer(ObjectStoreLayer::new(object_store_config)); Ok(self) diff --git a/core/lib/config/src/configs/fri_prover.rs b/core/lib/config/src/configs/fri_prover.rs index 2aa9a7fc89f3..99e3d354536e 100644 --- a/core/lib/config/src/configs/fri_prover.rs +++ b/core/lib/config/src/configs/fri_prover.rs @@ -26,7 +26,6 @@ pub struct FriProverConfig { // whether to write to public GCS bucket for https://github.com/matter-labs/era-boojum-validator-cli pub shall_save_to_public_bucket: bool, - pub core_object_store: Option, pub prover_object_store: Option, pub public_object_store: Option, } diff --git a/core/lib/config/src/configs/general.rs b/core/lib/config/src/configs/general.rs index ef02f557bc18..9f249d655f57 100644 --- a/core/lib/config/src/configs/general.rs +++ b/core/lib/config/src/configs/general.rs @@ -8,7 +8,8 @@ use crate::{ FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, }, - ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, PostgresConfig, SnapshotsCreatorConfig, + ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, ObjectStoreConfig, PostgresConfig, + SnapshotsCreatorConfig, }; #[derive(Debug)] @@ -34,4 +35,5 @@ pub struct GeneralConfig { pub snapshot_creator: Option, pub observability: Option, pub protective_reads_writer_config: Option, + pub core_object_store: Option, } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 458ed3390940..87c3bd2a1294 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -442,7 +442,6 @@ impl Distribution for EncodeDist { zone_read_url: self.sample(rng), shall_save_to_public_bucket: self.sample(rng), availability_check_interval_in_secs: self.sample(rng), - core_object_store: self.sample(rng), prover_object_store: self.sample(rng), public_object_store: self.sample(rng), } diff --git a/core/lib/env_config/src/fri_prover.rs b/core/lib/env_config/src/fri_prover.rs index f51bcd27d254..a6bd4e68385b 100644 --- a/core/lib/env_config/src/fri_prover.rs +++ b/core/lib/env_config/src/fri_prover.rs @@ -1,4 +1,4 @@ -use zksync_config::{configs::FriProverConfig, ObjectStoreConfig}; +use zksync_config::configs::FriProverConfig; use crate::{ envy_load, @@ -11,7 +11,6 @@ impl FromEnv for FriProverConfig { let mut prover: FriProverConfig = envy_load("fri_prover", "FRI_PROVER_")?; prover.prover_object_store = ProverObjectStoreConfig::from_env().map(|a| a.0).ok(); prover.public_object_store = PublicObjectStoreConfig::from_env().map(|a| a.0).ok(); - prover.core_object_store = ObjectStoreConfig::from_env().ok(); Ok(prover) } } diff --git a/core/lib/protobuf_config/src/general.rs b/core/lib/protobuf_config/src/general.rs index ba2076a09a14..834977759ae2 100644 --- a/core/lib/protobuf_config/src/general.rs +++ b/core/lib/protobuf_config/src/general.rs @@ -38,7 +38,9 @@ impl ProtoRepr for proto::GeneralConfig { .context("snapshot_creator")?, observability: read_optional_repr(&self.observability).context("observability")?, protective_reads_writer_config: read_optional_repr(&self.protective_reads_writer) - .context("vm_runner")?, + .context("protective_reads_writer")?, + core_object_store: read_optional_repr(&self.core_object_store) + .context("core_object_store")?, }) } @@ -74,6 +76,7 @@ impl ProtoRepr for proto::GeneralConfig { .protective_reads_writer_config .as_ref() .map(ProtoRepr::build), + core_object_store: this.core_object_store.as_ref().map(ProtoRepr::build), } } } diff --git a/core/lib/protobuf_config/src/proto/config/general.proto b/core/lib/protobuf_config/src/proto/config/general.proto index b606417d129a..fdfe257aecf1 100644 --- a/core/lib/protobuf_config/src/proto/config/general.proto +++ b/core/lib/protobuf_config/src/proto/config/general.proto @@ -14,6 +14,7 @@ import "zksync/config/observability.proto"; import "zksync/config/snapshots_creator.proto"; import "zksync/config/utils.proto"; import "zksync/config/vm_runner.proto"; +import "zksync/config/object_store.proto"; message GeneralConfig { optional config.database.Postgres postgres = 1; @@ -37,4 +38,5 @@ message GeneralConfig { optional config.snapshot_creator.SnapshotsCreator snapshot_creator = 31; optional config.observability.Observability observability = 32; optional config.vm_runner.ProtectiveReadsWriter protective_reads_writer = 33; + optional config.object_store.ObjectStore core_object_store = 34; } diff --git a/core/lib/protobuf_config/src/proto/config/prover.proto b/core/lib/protobuf_config/src/proto/config/prover.proto index a648dec78d31..d5d131fc157b 100644 --- a/core/lib/protobuf_config/src/proto/config/prover.proto +++ b/core/lib/protobuf_config/src/proto/config/prover.proto @@ -33,7 +33,6 @@ message Prover { optional string zone_read_url = 12; // required optional uint32 availability_check_interval_in_secs = 21; // optional; s optional bool shall_save_to_public_bucket = 13; // required - optional config.object_store.ObjectStore object_store = 20; optional config.object_store.ObjectStore public_object_store = 22; optional config.object_store.ObjectStore prover_object_store = 23; reserved 5, 6, 9; reserved "base_layer_circuit_ids_to_be_verified", "recursive_layer_circuit_ids_to_be_verified", "witness_vector_generator_thread_count"; diff --git a/core/lib/protobuf_config/src/prover.rs b/core/lib/protobuf_config/src/prover.rs index b917a4a6c89c..9a41e433433c 100644 --- a/core/lib/protobuf_config/src/prover.rs +++ b/core/lib/protobuf_config/src/prover.rs @@ -289,11 +289,6 @@ impl proto::SetupLoadMode { impl ProtoRepr for proto::Prover { type Type = configs::FriProverConfig; fn read(&self) -> anyhow::Result { - let core_object_store = if let Some(object_store) = &self.object_store { - Some(object_store.read()?) - } else { - None - }; let public_object_store = if let Some(object_store) = &self.public_object_store { Some(object_store.read()?) } else { @@ -335,7 +330,6 @@ impl ProtoRepr for proto::Prover { availability_check_interval_in_secs: self.availability_check_interval_in_secs, shall_save_to_public_bucket: *required(&self.shall_save_to_public_bucket) .context("shall_save_to_public_bucket")?, - core_object_store, public_object_store, prover_object_store, }) @@ -354,7 +348,6 @@ impl ProtoRepr for proto::Prover { zone_read_url: Some(this.zone_read_url.clone()), availability_check_interval_in_secs: this.availability_check_interval_in_secs, shall_save_to_public_bucket: Some(this.shall_save_to_public_bucket), - object_store: this.core_object_store.as_ref().map(ProtoRepr::build), prover_object_store: this.prover_object_store.as_ref().map(ProtoRepr::build), public_object_store: this.public_object_store.as_ref().map(ProtoRepr::build), } diff --git a/core/lib/zksync_core_leftovers/src/lib.rs b/core/lib/zksync_core_leftovers/src/lib.rs index 409117fb6b12..649a859cfd78 100644 --- a/core/lib/zksync_core_leftovers/src/lib.rs +++ b/core/lib/zksync_core_leftovers/src/lib.rs @@ -472,9 +472,6 @@ pub async fn initialize_components( } let object_store_config = configs - .prover_config - .clone() - .context("Prover")? .core_object_store .clone() .context("core_object_store_config")?; diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 65c7148d2747..1f4c410ed9c1 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -13,7 +13,7 @@ use zksync_config::{ ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, - PostgresConfig, SnapshotsCreatorConfig, + ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, }; use zksync_protobuf::{repr::ProtoRepr, ProtoFmt}; @@ -61,6 +61,7 @@ pub struct TempConfigStore { pub observability: Option, pub snapshot_creator: Option, pub protective_reads_writer_config: Option, + pub core_object_store: Option, } impl TempConfigStore { @@ -87,6 +88,7 @@ impl TempConfigStore { snapshot_creator: self.snapshot_creator.clone(), observability: self.observability.clone(), protective_reads_writer_config: self.protective_reads_writer_config.clone(), + core_object_store: self.core_object_store.clone(), } } diff --git a/prover/config/src/lib.rs b/prover/config/src/lib.rs index b21cb1c4be41..f501dd2d6e06 100644 --- a/prover/config/src/lib.rs +++ b/prover/config/src/lib.rs @@ -10,7 +10,8 @@ use zksync_config::{ house_keeper::HouseKeeperConfig, DatabaseSecrets, FriProofCompressorConfig, FriProverConfig, FriProverGatewayConfig, FriWitnessGeneratorConfig, FriWitnessVectorGeneratorConfig, GeneralConfig, - ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, ProtectiveReadsWriterConfig, + ObjectStoreConfig, ObservabilityConfig, PrometheusConfig, ProofDataHandlerConfig, + ProtectiveReadsWriterConfig, }, ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, PostgresConfig, SnapshotsCreatorConfig, @@ -48,6 +49,7 @@ fn load_env_config() -> anyhow::Result { observability: ObservabilityConfig::from_env().ok(), snapshot_creator: SnapshotsCreatorConfig::from_env().ok(), protective_reads_writer_config: ProtectiveReadsWriterConfig::from_env().ok(), + core_object_store: ObjectStoreConfig::from_env().ok(), }) }