Merge branch 'main' into bft-486-bump-cons-ver

core/bin/snapshots_creator/Cargo.toml

@@ -19,8 +19,10 @@ zksync_env_config.workspace = true
 zksync_types.workspace = true
 zksync_object_store.workspace = true
 zksync_vlog.workspace = true
+zksync_core_leftovers.workspace = true
 
 anyhow.workspace = true
+structopt.workspace = true
 tokio = { workspace = true, features = ["full"] }
 tracing.workspace = true
 futures.workspace = true

core/bin/snapshots_creator/src/main.rs

@@ -10,13 +10,11 @@
 //! at a time).
 
 use anyhow::Context as _;
+use structopt::StructOpt;
 use tokio::{sync::watch, task::JoinHandle};
-use zksync_config::{
-    configs::{DatabaseSecrets, ObservabilityConfig, PrometheusConfig},
-    SnapshotsCreatorConfig,
-};
+use zksync_config::configs::PrometheusConfig;
+use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config};
 use zksync_dal::{ConnectionPool, Core};
-use zksync_env_config::{object_store::SnapshotsObjectStoreConfig, FromEnv};
 use zksync_object_store::ObjectStoreFactory;
 use zksync_vlog::prometheus::PrometheusExporterConfig;
 
@@ -28,9 +26,9 @@ mod metrics;
 mod tests;
 
 async fn maybe_enable_prometheus_metrics(
+    prometheus_config: Option<PrometheusConfig>,
     stop_receiver: watch::Receiver<bool>,
 ) -> anyhow::Result<Option<JoinHandle<anyhow::Result<()>>>> {
-    let prometheus_config = PrometheusConfig::from_env().ok();
     match prometheus_config.map(|c| (c.gateway_endpoint(), c.push_interval())) {
         Some((Some(gateway_endpoint), push_interval)) => {
             tracing::info!("Starting prometheus exporter with gateway {gateway_endpoint:?} and push_interval {push_interval:?}");
@@ -49,18 +47,36 @@ async fn maybe_enable_prometheus_metrics(
 /// Minimum number of storage log chunks to produce.
 const MIN_CHUNK_COUNT: u64 = 10;
 
+#[derive(StructOpt)]
+#[structopt(name = "ZKsync snapshot creator", author = "Matter Labs")]
+struct Opt {
+    /// Path to the configuration file.
+    #[structopt(long)]
+    config_path: Option<std::path::PathBuf>,
+
+    /// Path to the secrets file.
+    #[structopt(long)]
+    secrets_path: Option<std::path::PathBuf>,
+}
+
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     let (stop_sender, stop_receiver) = watch::channel(false);
 
-    let observability_config =
-        ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?;
+    let opt = Opt::from_args();
+    let general_config = load_general_config(opt.config_path).context("general config")?;
+    let database_secrets = load_database_secrets(opt.secrets_path).context("database secrets")?;
+
+    let observability_config = general_config
+        .observability
+        .context("observability config")?;
     let log_format: zksync_vlog::LogFormat = observability_config
         .log_format
         .parse()
         .context("Invalid log format")?;
 
-    let prometheus_exporter_task = maybe_enable_prometheus_metrics(stop_receiver).await?;
+    let prometheus_exporter_task =
+        maybe_enable_prometheus_metrics(general_config.prometheus_config, stop_receiver).await?;
     let mut builder = zksync_vlog::ObservabilityBuilder::new().with_log_format(log_format);
     if let Some(sentry_url) = observability_config.sentry_url {
         builder = builder
@@ -71,16 +87,19 @@ async fn main() -> anyhow::Result<()> {
     let _guard = builder.build();
     tracing::info!("Starting snapshots creator");
 
-    let object_store_config =
-        SnapshotsObjectStoreConfig::from_env().context("SnapshotsObjectStoreConfig::from_env()")?;
-    let blob_store = ObjectStoreFactory::new(object_store_config.0)
+    let creator_config = general_config
+        .snapshot_creator
+        .context("snapshot creator config")?;
+
+    let object_store_config = creator_config
+        .clone()
+        .object_store
+        .context("snapshot creator object storage config")?;
+
+    let blob_store = ObjectStoreFactory::new(object_store_config)
         .create_store()
         .await?;
 
-    let database_secrets = DatabaseSecrets::from_env().context("DatabaseSecrets")?;
-    let creator_config =
-        SnapshotsCreatorConfig::from_env().context("SnapshotsCreatorConfig::from_env")?;
-
     let replica_pool = ConnectionPool::<Core>::builder(
         database_secrets.replica_url()?,
         creator_config.concurrent_queries_count,

core/bin/snapshots_creator/src/tests.rs

@@ -10,6 +10,7 @@ use std::{
 };
 
 use rand::{thread_rng, Rng};
+use zksync_config::SnapshotsCreatorConfig;
 use zksync_dal::{Connection, CoreDal};
 use zksync_object_store::{MockObjectStore, ObjectStore};
 use zksync_types::{

core/lib/env_config/src/snapshots_creator.rs

@@ -1,9 +1,13 @@
 use zksync_config::SnapshotsCreatorConfig;
 
-use crate::{envy_load, FromEnv};
+use crate::{envy_load, object_store::SnapshotsObjectStoreConfig, FromEnv};
 
 impl FromEnv for SnapshotsCreatorConfig {
     fn from_env() -> anyhow::Result<Self> {
-        envy_load("snapshots_creator", "SNAPSHOTS_CREATOR_")
+        let mut snapshot_creator: SnapshotsCreatorConfig =
+            envy_load("snapshots_creator", "SNAPSHOTS_CREATOR_")?;
+
+        snapshot_creator.object_store = SnapshotsObjectStoreConfig::from_env().map(|a| a.0).ok();
+        Ok(snapshot_creator)
     }
 }

docs/guides/external-node/09_decentralization.md

@@ -17,11 +17,10 @@ On the gossipnet, the data integrity will be protected by the BFT (byzantine fau
 > current implementation it may take a couple of hours and gets faster the more nodes you add to the
 > `gossip_static_outbound` list (see below). We are working to remove this inconvenience.
 
-
 > [!NOTE]
 >
-> The minimal supported server version for this is [24.11.0](https://github.com/matter-labs/zksync-era/releases/tag/core-v24.11.0)
-
+> The minimal supported server version for this is
+> [24.11.0](https://github.com/matter-labs/zksync-era/releases/tag/core-v24.11.0)
 
 ### Generating secrets
 

zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/create.rs

@@ -1,20 +1,23 @@
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
+use anyhow::bail;
 use clap::Parser;
-use common::{Prompt, PromptConfirm, PromptSelect};
+use common::{cmd::Cmd, logger, Prompt, PromptConfirm, PromptSelect};
 use serde::{Deserialize, Serialize};
 use slugify_rs::slugify;
 use strum::IntoEnumIterator;
 use strum_macros::EnumIter;
 use types::{L1Network, WalletCreation};
+use xshell::{cmd, Shell};
 
 use crate::{
     commands::chain::{args::create::ChainCreateArgs, ChainCreateArgsFinal},
     messages::{
+        msg_path_to_zksync_does_not_exist_err, MSG_CONFIRM_STILL_USE_FOLDER,
         MSG_ECOSYSTEM_NAME_PROMPT, MSG_L1_NETWORK_HELP, MSG_L1_NETWORK_PROMPT,
         MSG_LINK_TO_CODE_HELP, MSG_LINK_TO_CODE_PROMPT, MSG_LINK_TO_CODE_SELECTION_CLONE,
-        MSG_LINK_TO_CODE_SELECTION_PATH, MSG_REPOSITORY_ORIGIN_PROMPT, MSG_START_CONTAINERS_HELP,
-        MSG_START_CONTAINERS_PROMPT,
+        MSG_LINK_TO_CODE_SELECTION_PATH, MSG_NOT_MAIN_REPO_OR_FORK_ERR,
+        MSG_REPOSITORY_ORIGIN_PROMPT, MSG_START_CONTAINERS_HELP, MSG_START_CONTAINERS_PROMPT,
     },
 };
 
@@ -34,7 +37,7 @@ pub struct EcosystemCreateArgs {
 }
 
 impl EcosystemCreateArgs {
-    pub fn fill_values_with_prompt(mut self) -> EcosystemCreateArgsFinal {
+    pub fn fill_values_with_prompt(mut self, shell: &Shell) -> EcosystemCreateArgsFinal {
         let mut ecosystem_name = self
             .ecosystem_name
             .unwrap_or_else(|| Prompt::new(MSG_ECOSYSTEM_NAME_PROMPT).ask());
@@ -45,7 +48,16 @@ impl EcosystemCreateArgs {
                 PromptSelect::new(MSG_REPOSITORY_ORIGIN_PROMPT, LinkToCodeSelection::iter()).ask();
             match link_to_code_selection {
                 LinkToCodeSelection::Clone => "".to_string(),
-                LinkToCodeSelection::Path => Prompt::new(MSG_LINK_TO_CODE_PROMPT).ask(),
+                LinkToCodeSelection::Path => {
+                    let mut path: String = Prompt::new(MSG_LINK_TO_CODE_PROMPT).ask();
+                    if let Err(err) = check_link_to_code(shell, &path) {
+                        logger::warn(err);
+                        if !PromptConfirm::new(MSG_CONFIRM_STILL_USE_FOLDER).ask() {
+                            path = pick_new_link_to_code(shell);
+                        }
+                    }
+                    path
+                }
             }
         });
 
@@ -105,3 +117,40 @@ impl std::fmt::Display for LinkToCodeSelection {
         }
     }
 }
+
+fn check_link_to_code(shell: &Shell, path: &str) -> anyhow::Result<()> {
+    let path = Path::new(path);
+    if !shell.path_exists(path) {
+        bail!(msg_path_to_zksync_does_not_exist_err(
+            path.to_str().unwrap()
+        ));
+    }
+
+    let _guard = shell.push_dir(path);
+    let out = String::from_utf8(
+        Cmd::new(cmd!(shell, "git remote -v"))
+            .run_with_output()?
+            .stdout,
+    )?;
+
+    if !out.contains("matter-labs/zksync-era") {
+        bail!(MSG_NOT_MAIN_REPO_OR_FORK_ERR);
+    }
+
+    Ok(())
+}
+
+fn pick_new_link_to_code(shell: &Shell) -> String {
+    let link_to_code: String = Prompt::new(MSG_LINK_TO_CODE_PROMPT).ask();
+    match check_link_to_code(shell, &link_to_code) {
+        Ok(_) => link_to_code,
+        Err(err) => {
+            logger::warn(err);
+            if !PromptConfirm::new(MSG_CONFIRM_STILL_USE_FOLDER).ask() {
+                pick_new_link_to_code(shell)
+            } else {
+                link_to_code
+            }
+        }
+    }
+}

zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs

@@ -39,7 +39,7 @@ pub fn run(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> {
 }
 
 fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> {
-    let args = args.fill_values_with_prompt();
+    let args = args.fill_values_with_prompt(shell);
 
     logger::note(MSG_SELECTED_CONFIG, logger::object_to_string(&args));
     logger::info(MSG_CREATING_ECOSYSTEM);

zk_toolbox/crates/zk_inception/src/messages.rs

@@ -36,6 +36,13 @@ pub(super) const MSG_ECOSYSTEM_ALREADY_EXISTS_ERR: &str = "Ecosystem already exi
 pub(super) const MSG_ECOSYSTEM_CONFIG_INVALID_ERR: &str = "Invalid ecosystem configuration";
 pub(super) const MSG_LINK_TO_CODE_SELECTION_CLONE: &str = "Clone for me (recommended)";
 pub(super) const MSG_LINK_TO_CODE_SELECTION_PATH: &str = "I have the code already";
+pub(super) const MSG_NOT_MAIN_REPO_OR_FORK_ERR: &str =
+    "It's not a zkSync Era main repository or fork";
+pub(super) const MSG_CONFIRM_STILL_USE_FOLDER: &str = "Do you still want to use this folder?";
+
+pub(super) fn msg_path_to_zksync_does_not_exist_err(path: &str) -> String {
+    format!("Path to zkSync Era repo does not exist: {path:?}")
+}
 
 /// Ecosystem and chain init related messages
 pub(super) const MSG_L1_RPC_URL_HELP: &str = "L1 RPC URL";