feat(zk_toolbox): Add grafana support

zk_toolbox/crates/config/src/consts.rs

@@ -24,6 +24,12 @@ pub const ZKSYNC_ERA_GIT_REPO: &str = "https://github.com/matter-labs/zksync-era
 pub const DOCKER_COMPOSE_FILE: &str = "docker-compose.yml";
 /// Path to the config file with mnemonic for localhost wallets
 pub(crate) const CONFIGS_PATH: &str = "etc/env/file_based";
+/// Path to the docker-compose file for grafana
+pub const ERA_OBSERVABILITY_COMPOSE_FILE: &str = "era-observability/docker-compose.yml";
+/// Path to era observability repository
+pub const ERA_OBSERBAVILITY_DIR: &str = "era-observability";
+/// Era observability repo link
+pub const ERA_OBSERBAVILITY_GIT_REPO: &str = "https://github.com/matter-labs/era-observability";
 pub(crate) const LOCAL_CONFIGS_PATH: &str = "configs/";
 pub(crate) const LOCAL_DB_PATH: &str = "db/";
 

zk_toolbox/crates/zk_inception/src/commands/args/containers.rs

@@ -0,0 +1,9 @@
+use clap::Parser;
+
+use crate::messages::MSG_RUN_OBSERVABILITY_HELP;
+
+#[derive(Debug, Parser)]
+pub struct ContainersArgs {
+    #[clap(long, short = 'o', help = MSG_RUN_OBSERVABILITY_HELP)]
+    pub run_observability: bool,
+}

zk_toolbox/crates/zk_inception/src/commands/args/mod.rs

@@ -1,5 +1,7 @@
+pub use containers::*;
 pub use run_server::*;
 pub use update::*;
 
+mod containers;
 mod run_server;
 mod update;

zk_toolbox/crates/zk_inception/src/commands/containers.rs

@@ -2,23 +2,24 @@ use std::path::PathBuf;
 
 use anyhow::{anyhow, Context};
 use common::{docker, logger, spinner::Spinner};
-use config::{EcosystemConfig, DOCKER_COMPOSE_FILE};
+use config::{EcosystemConfig, DOCKER_COMPOSE_FILE, ERA_OBSERVABILITY_COMPOSE_FILE};
 use xshell::Shell;
 
+use super::args::ContainersArgs;
 use crate::messages::{
     MSG_CONTAINERS_STARTED, MSG_FAILED_TO_FIND_ECOSYSTEM_ERR, MSG_RETRY_START_CONTAINERS_PROMPT,
     MSG_STARTING_CONTAINERS, MSG_STARTING_DOCKER_CONTAINERS_SPINNER,
 };
 
-pub fn run(shell: &Shell) -> anyhow::Result<()> {
+pub fn run(shell: &Shell, args: ContainersArgs) -> anyhow::Result<()> {
     let ecosystem = EcosystemConfig::from_file(shell).context(MSG_FAILED_TO_FIND_ECOSYSTEM_ERR)?;
 
     initialize_docker(shell, &ecosystem)?;
 
     logger::info(MSG_STARTING_CONTAINERS);
 
     let spinner = Spinner::new(MSG_STARTING_DOCKER_CONTAINERS_SPINNER);
-    start_containers(shell)?;
+    start_containers(shell, args.run_observability)?;
     spinner.finish();
 
     logger::outro(MSG_CONTAINERS_STARTED);
@@ -37,19 +38,35 @@ pub fn initialize_docker(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::
     Ok(())
 }
 
-pub fn start_containers(shell: &Shell) -> anyhow::Result<()> {
-    while let Err(err) = docker::up(shell, DOCKER_COMPOSE_FILE) {
+fn start_container(shell: &Shell, compose_file: &str, retry_msg: &str) -> anyhow::Result<()> {
+    while let Err(err) = docker::up(shell, compose_file) {
         logger::error(err.to_string());
-        if !common::PromptConfirm::new(MSG_RETRY_START_CONTAINERS_PROMPT)
-            .default(true)
-            .ask()
-        {
+        if !common::PromptConfirm::new(retry_msg).default(true).ask() {
             return Err(err);
         }
     }
     Ok(())
 }
 
+pub fn start_containers(shell: &Shell, run_observability: bool) -> anyhow::Result<()> {
+    start_container(
+        shell,
+        DOCKER_COMPOSE_FILE,
+        MSG_RETRY_START_CONTAINERS_PROMPT,
+    )?;
+
+    if !run_observability {
+        return Ok(());
+    }
+    start_container(
+        shell,
+        ERA_OBSERVABILITY_COMPOSE_FILE,
+        MSG_RETRY_START_CONTAINERS_PROMPT,
+    )?;
+
+    Ok(())
+}
+
 fn create_docker_folders(shell: &Shell) -> anyhow::Result<()> {
     shell.create_dir("volumes")?;
     shell.create_dir("volumes/postgres")?;

zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs

@@ -12,7 +12,7 @@ use crate::{
     messages::{
         MSG_DEPLOY_ECOSYSTEM_PROMPT, MSG_DEPLOY_ERC20_PROMPT, MSG_DEPLOY_PAYMASTER_PROMPT,
         MSG_DEV_ARG_HELP, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR,
-        MSG_L1_RPC_URL_PROMPT,
+        MSG_L1_RPC_URL_PROMPT, MSG_RUN_OBSERVABILITY_HELP,
     },
 };
 
@@ -90,6 +90,8 @@ pub struct EcosystemInitArgs {
     pub genesis_args: GenesisArgs,
     #[clap(long, help = MSG_DEV_ARG_HELP)]
     pub dev: bool,
+    #[clap(long, short = 'o', help = MSG_RUN_OBSERVABILITY_HELP)]
+    pub run_observability: bool,
 }
 
 impl EcosystemInitArgs {
@@ -117,6 +119,7 @@ impl EcosystemInitArgs {
             ecosystem,
             forge_args: self.forge_args.clone(),
             dev: self.dev,
+            run_observability: self.run_observability,
         }
     }
 }
@@ -128,4 +131,5 @@ pub struct EcosystemInitArgsFinal {
     pub ecosystem: EcosystemArgsFinal,
     pub forge_args: ForgeScriptArgs,
     pub dev: bool,
+    pub run_observability: bool,
 }

zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs

@@ -107,7 +107,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> {
     if args.start_containers {
         let spinner = Spinner::new(MSG_STARTING_CONTAINERS_SPINNER);
         initialize_docker(shell, &ecosystem_config)?;
-        start_containers(shell)?;
+        start_containers(shell, false)?;
         spinner.finish();
     }
 

zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs

@@ -26,7 +26,8 @@ use config::{
         FileConfigWithDefaultName, ReadConfig, ReadConfigWithBasePath, SaveConfig,
         SaveConfigWithBasePath,
     },
-    ChainConfig, ContractsConfig, EcosystemConfig, GenesisConfig,
+    ChainConfig, ContractsConfig, EcosystemConfig, GenesisConfig, ERA_OBSERBAVILITY_DIR,
+    ERA_OBSERBAVILITY_GIT_REPO,
 };
 use types::{L1Network, ProverMode, WalletCreation};
 use xshell::{cmd, Shell};
@@ -45,7 +46,8 @@ use crate::{
         msg_ecosystem_initialized, msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED,
         MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20,
         MSG_DEPLOYING_ERC20_SPINNER, MSG_DISTRIBUTING_ETH_SPINNER,
-        MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT,
+        MSG_DOWNLOADING_ERA_OBSERVABILITY_SPINNER, MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR,
+        MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, MSG_ERA_OBSERVABILITY_ALREADY_SETUP,
         MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER,
     },
     utils::forge::{check_the_balance, fill_forge_private_key},
@@ -69,6 +71,10 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> {
 
     logger::info(MSG_INITIALIZING_ECOSYSTEM);
 
+    if final_ecosystem_args.run_observability {
+        download_observability(shell)?;
+    }
+
     let contracts_config = init(
         &mut final_ecosystem_args,
         shell,
@@ -380,3 +386,23 @@ fn build_system_contracts(shell: &Shell, link_to_code: &Path) -> anyhow::Result<
     let _dir_guard = shell.push_dir(link_to_code.join("contracts"));
     Ok(Cmd::new(cmd!(shell, "yarn sc build")).run()?)
 }
+
+/// Downloads Grafana dashboards from the era-observability repo
+fn download_observability(shell: &Shell) -> anyhow::Result<()> {
+    let path_to_era_observability = shell.current_dir().join(ERA_OBSERBAVILITY_DIR);
+    if shell.path_exists(path_to_era_observability.clone()) {
+        logger::info(MSG_ERA_OBSERVABILITY_ALREADY_SETUP);
+        return Ok(());
+    }
+
+    let spinner = Spinner::new(MSG_DOWNLOADING_ERA_OBSERVABILITY_SPINNER);
+    git::clone(
+        shell,
+        shell.current_dir(),
+        ERA_OBSERBAVILITY_GIT_REPO,
+        ERA_OBSERBAVILITY_DIR,
+    )?;
+    spinner.finish();
+
+    Ok(())
+}

zk_toolbox/crates/zk_inception/src/commands/update.rs

@@ -1,14 +1,10 @@
 use std::path::Path;
 
 use anyhow::{Context, Ok};
-use common::{
-    git::{pull, submodule_update},
-    logger,
-    spinner::Spinner,
-};
+use common::{git, logger, spinner::Spinner};
 use config::{
-    ChainConfig, EcosystemConfig, CONTRACTS_FILE, EN_CONFIG_FILE, GENERAL_FILE, GENESIS_FILE,
-    SECRETS_FILE,
+    ChainConfig, EcosystemConfig, CONTRACTS_FILE, EN_CONFIG_FILE, ERA_OBSERBAVILITY_DIR,
+    GENERAL_FILE, GENESIS_FILE, SECRETS_FILE,
 };
 use xshell::Shell;
 
@@ -17,7 +13,8 @@ use crate::messages::{
     msg_diff_contracts_config, msg_diff_genesis_config, msg_diff_secrets, msg_updating_chain,
     MSG_CHAIN_NOT_FOUND_ERR, MSG_DIFF_EN_CONFIG, MSG_DIFF_EN_GENERAL_CONFIG,
     MSG_DIFF_GENERAL_CONFIG, MSG_INVALID_KEY_TYPE_ERR, MSG_PULLING_ZKSYNC_CODE_SPINNER,
-    MSG_UPDATING_SUBMODULES_SPINNER, MSG_UPDATING_ZKSYNC, MSG_ZKSYNC_UPDATED,
+    MSG_UPDATING_ERA_OBSERVABILITY_SPINNER, MSG_UPDATING_SUBMODULES_SPINNER, MSG_UPDATING_ZKSYNC,
+    MSG_ZKSYNC_UPDATED,
 };
 
 /// Holds the differences between two YAML configurations.
@@ -77,6 +74,13 @@ pub fn run(shell: &Shell, args: UpdateArgs) -> anyhow::Result<()> {
         )?;
     }
 
+    let path_to_era_observability = shell.current_dir().join(ERA_OBSERBAVILITY_DIR);
+    if shell.path_exists(path_to_era_observability.clone()) {
+        let spinner = Spinner::new(MSG_UPDATING_ERA_OBSERVABILITY_SPINNER);
+        git::pull(shell, path_to_era_observability)?;
+        spinner.finish();
+    }
+
     logger::outro(MSG_ZKSYNC_UPDATED);
 
     Ok(())
@@ -86,10 +90,10 @@ fn update_repo(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()>
     let link_to_code = ecosystem.link_to_code.clone();
 
     let spinner = Spinner::new(MSG_PULLING_ZKSYNC_CODE_SPINNER);
-    pull(shell, link_to_code.clone())?;
+    git::pull(shell, link_to_code.clone())?;
     spinner.finish();
     let spinner = Spinner::new(MSG_UPDATING_SUBMODULES_SPINNER);
-    submodule_update(shell, link_to_code.clone())?;
+    git::submodule_update(shell, link_to_code.clone())?;
     spinner.finish();
 
     Ok(())

zk_toolbox/crates/zk_inception/src/main.rs

@@ -1,5 +1,8 @@
 use clap::{command, Parser, Subcommand};
-use commands::{args::UpdateArgs, contract_verifier::ContractVerifierCommands};
+use commands::{
+    args::{ContainersArgs, UpdateArgs},
+    contract_verifier::ContractVerifierCommands,
+};
 use common::{
     check_general_prerequisites,
     config::{global_config, init_global_config, GlobalConfig},
@@ -49,7 +52,7 @@ pub enum InceptionSubcommands {
     ExternalNode(ExternalNodeCommands),
     /// Run containers for local development
     #[command(alias = "up")]
-    Containers,
+    Containers(ContainersArgs),
     /// Run contract verifier
     #[command(subcommand)]
     ContractVerifier(ContractVerifierCommands),
@@ -106,7 +109,7 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res
         InceptionSubcommands::Chain(args) => commands::chain::run(shell, args).await?,
         InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?,
         InceptionSubcommands::Server(args) => commands::server::run(shell, args)?,
-        InceptionSubcommands::Containers => commands::containers::run(shell)?,
+        InceptionSubcommands::Containers(args) => commands::containers::run(shell, args)?,
         InceptionSubcommands::ExternalNode(args) => {
             commands::external_node::run(shell, args).await?
         }

zk_toolbox/crates/zk_inception/src/messages.rs

@@ -49,6 +49,7 @@ pub(super) const MSG_L1_RPC_URL_HELP: &str = "L1 RPC URL";
 pub(super) const MSG_GENESIS_ARGS_HELP: &str = "Genesis options";
 pub(super) const MSG_DEV_ARG_HELP: &str =
     "Deploy ecosystem  using all defaults. Suitable for local development";
+pub(super) const MSG_RUN_OBSERVABILITY_HELP: &str = "Enable Grafana";
 pub(super) const MSG_DEPLOY_ECOSYSTEM_PROMPT: &str =
     "Do you want to deploy ecosystem contracts? (Not needed if you already have an existing one)";
 pub(super) const MSG_L1_RPC_URL_PROMPT: &str = "What is the RPC URL of the L1 network?";
@@ -71,6 +72,9 @@ pub(super) const MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER: &str =
 pub(super) const MSG_REGISTERING_CHAIN_SPINNER: &str = "Registering chain...";
 pub(super) const MSG_ACCEPTING_ADMIN_SPINNER: &str = "Accepting admin...";
 pub(super) const MSG_RECREATE_ROCKS_DB_ERRROR: &str = "Failed to create rocks db path";
+pub(super) const MSG_ERA_OBSERVABILITY_ALREADY_SETUP: &str = "Era observability already setup";
+pub(super) const MSG_DOWNLOADING_ERA_OBSERVABILITY_SPINNER: &str =
+    "Downloading era observability...";
 
 pub(super) fn msg_initializing_chain(chain_name: &str) -> String {
     format!("Initializing chain {chain_name}")
@@ -337,6 +341,7 @@ pub(super) const MSG_DIFF_EN_CONFIG: &str =
 pub(super) const MSG_DIFF_EN_GENERAL_CONFIG: &str =
     "Added the following fields to the external node generalconfig:";
 pub(super) const MSG_INVALID_KEY_TYPE_ERR: &str = "Invalid key type";
+pub(super) const MSG_UPDATING_ERA_OBSERVABILITY_SPINNER: &str = "Updating era observability...";
 
 pub(super) fn msg_diff_genesis_config(chain: &str) -> String {
     format!(