From f3efb311b6cf6a293446009827f69256f6deb801 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 12 Jun 2024 12:33:41 +0200 Subject: [PATCH 01/87] Add init command boilerplate --- .../crates/zk_inception/src/commands/mod.rs | 1 + .../src/commands/prover/args/init.rs | 5 +++++ .../src/commands/prover/args/mod.rs | 1 + .../zk_inception/src/commands/prover/init.rs | 10 ++++++++++ .../zk_inception/src/commands/prover/mod.rs | 17 +++++++++++++++++ zk_toolbox/crates/zk_inception/src/main.rs | 8 +++++++- 6 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/init.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index 8ed7a82b8334..ccdf5b082caa 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -2,4 +2,5 @@ pub mod args; pub mod chain; pub mod containers; pub mod ecosystem; +pub mod prover; pub mod server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs new file mode 100644 index 000000000000..8a8709b69e25 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs @@ -0,0 +1,5 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, Parser)] +pub struct InitArgs {} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs new file mode 100644 index 000000000000..43763f10a418 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs @@ -0,0 +1 @@ +pub mod init; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs new file mode 100644 index 000000000000..3792d45a63a9 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -0,0 +1,10 @@ +use anyhow::Ok; +use common::logger; +use xshell::Shell; + +use super::args::init::InitArgs; + +pub(crate) async fn run(_args: InitArgs, _shell: &Shell) -> anyhow::Result<()> { + logger::debug("Initializing prover"); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs new file mode 100644 index 000000000000..fef963f624eb --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -0,0 +1,17 @@ +use args::init::InitArgs; +use clap::Subcommand; +use xshell::Shell; +mod args; +mod init; + +#[derive(Subcommand, Debug)] +pub enum ProverCommands { + /// Initialize prover + Init(InitArgs), +} + +pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { + match args { + ProverCommands::Init(args) => init::run(args, shell).await, + } +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index b0e8e8f4fd69..dff9e479e01f 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -7,7 +7,9 @@ use common::{ use config::EcosystemConfig; use xshell::Shell; -use crate::commands::{args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands}; +use crate::commands::{ + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, prover::ProverCommands, +}; pub mod accept_ownership; mod commands; @@ -35,6 +37,9 @@ pub enum InceptionSubcommands { /// Chain related commands #[command(subcommand)] Chain(ChainCommands), + /// Prover related commands + #[command(subcommand)] + Prover(ProverCommands), /// Run server Server(RunServerArgs), /// Run containers for local development @@ -101,6 +106,7 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res match inception_args.command { InceptionSubcommands::Ecosystem(args) => commands::ecosystem::run(shell, args).await?, InceptionSubcommands::Chain(args) => commands::chain::run(shell, args).await?, + InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, } From d82b8b30700fc266948d72de4f68fbc419e1c6b4 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 13 Jun 2024 12:46:13 +0200 Subject: [PATCH 02/87] Run key generator --- .../zk_inception/src/commands/prover/init.rs | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 3792d45a63a9..4e491882657b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -1,10 +1,29 @@ use anyhow::Ok; -use common::logger; -use xshell::Shell; +use common::{cmd::Cmd, logger, spinner::Spinner}; +use config::EcosystemConfig; +use xshell::{cmd, Shell}; use super::args::init::InitArgs; -pub(crate) async fn run(_args: InitArgs, _shell: &Shell) -> anyhow::Result<()> { - logger::debug("Initializing prover"); +pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { + logger::info("Initializing prover"); + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let mut link_to_prover = ecosystem_config.link_to_code.into_os_string(); + link_to_prover.push("/prover"); + shell.change_dir(link_to_prover); + + let spinner = Spinner::new("Generating setup keys..."); + + let mut cmd = Cmd::new(cmd!( + shell, + "cargo run --features gpu --release --bin key_generator -- + generate-sk all --recompute-if-missing + --setup-path=vk_setup_data_generator_server_fri/data + --path=prover/vk_setup_data_generator_server_fri/data" + )); + cmd.run()?; + spinner.finish(); + logger::info("Setup keys generated successfully"); + Ok(()) } From 351c9eeed31a45de64b93034b1eacfa936b6c0df Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 13 Jun 2024 13:05:46 +0200 Subject: [PATCH 03/87] fix setup data path --- .../crates/zk_inception/src/commands/prover/init.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 4e491882657b..90325072eb50 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -10,20 +10,19 @@ pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; let mut link_to_prover = ecosystem_config.link_to_code.into_os_string(); link_to_prover.push("/prover"); - shell.change_dir(link_to_prover); - - let spinner = Spinner::new("Generating setup keys..."); + shell.change_dir(link_to_prover.clone()); + let spinner = Spinner::new("Generating verification keys..."); let mut cmd = Cmd::new(cmd!( shell, "cargo run --features gpu --release --bin key_generator -- generate-sk all --recompute-if-missing --setup-path=vk_setup_data_generator_server_fri/data - --path=prover/vk_setup_data_generator_server_fri/data" + --path={link_to_prover}/vk_setup_data_generator_server_fri/data" )); cmd.run()?; spinner.finish(); - logger::info("Setup keys generated successfully"); + logger::info("Verification keys generated successfully"); Ok(()) } From 866bdc9a513f73213d75f9eb4459be0c0edfed7d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 13 Jun 2024 13:28:20 +0200 Subject: [PATCH 04/87] Refactor msgs --- .../crates/zk_inception/src/commands/prover/init.rs | 8 +++++--- zk_toolbox/crates/zk_inception/src/messages.rs | 5 +++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 90325072eb50..9a9d40d5a38b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -3,16 +3,18 @@ use common::{cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; +use crate::messages::{MSG_GENERATING_VK_SPINNER, MSG_INITIALIZING_PROVER, MSG_PROVER_INITIALIZED}; + use super::args::init::InitArgs; pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { - logger::info("Initializing prover"); + logger::info(MSG_INITIALIZING_PROVER); let ecosystem_config = EcosystemConfig::from_file(shell)?; let mut link_to_prover = ecosystem_config.link_to_code.into_os_string(); link_to_prover.push("/prover"); shell.change_dir(link_to_prover.clone()); - let spinner = Spinner::new("Generating verification keys..."); + let spinner = Spinner::new(MSG_GENERATING_VK_SPINNER); let mut cmd = Cmd::new(cmd!( shell, "cargo run --features gpu --release --bin key_generator -- @@ -22,7 +24,7 @@ pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { )); cmd.run()?; spinner.finish(); - logger::info("Verification keys generated successfully"); + logger::outro(MSG_PROVER_INITIALIZED); Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 7221f030d415..884884062a14 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -173,3 +173,8 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt(address: &H160) -> Str "Address {address:?} doesn't have enough money to deploy contracts do you want to try again?" ) } + +/// Prover related messages +pub(super) const MSG_INITIALIZING_PROVER: &str = "Initializing prover"; +pub(super) const MSG_PROVER_INITIALIZED: &str = "Prover initialized successfully"; +pub(super) const MSG_GENERATING_VK_SPINNER: &str = "Generating verification keys..."; From 535fd18e127159a9fbac39d8f57a87ac79482eed Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 13 Jun 2024 13:28:47 +0200 Subject: [PATCH 05/87] fmt --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 9a9d40d5a38b..f82b798b5cba 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -3,9 +3,8 @@ use common::{cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; -use crate::messages::{MSG_GENERATING_VK_SPINNER, MSG_INITIALIZING_PROVER, MSG_PROVER_INITIALIZED}; - use super::args::init::InitArgs; +use crate::messages::{MSG_GENERATING_VK_SPINNER, MSG_INITIALIZING_PROVER, MSG_PROVER_INITIALIZED}; pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { logger::info(MSG_INITIALIZING_PROVER); From 29e4459926b8c4f29df61b37c56029fac1a6aa88 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Fri, 14 Jun 2024 12:32:59 +0200 Subject: [PATCH 06/87] Save link to prover in ecosystem config --- zk_toolbox/crates/config/src/ecosystem.rs | 12 ++++++++++++ .../zk_inception/src/commands/ecosystem/create.rs | 6 +++++- .../crates/zk_inception/src/commands/prover/init.rs | 5 ++--- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 1557ab21646f..932e1177b10f 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -25,6 +25,7 @@ struct EcosystemConfigInternal { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, + pub link_to_prover: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -40,6 +41,7 @@ pub struct EcosystemConfig { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, + pub link_to_prover: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -72,6 +74,11 @@ impl<'de> Deserialize<'de> for EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .to_path_buf(), + link_to_prover: config + .link_to_prover + .absolutize() + .expect("Failed to parse prover path") + .to_path_buf(), chains: config.chains.clone(), config: config.config.clone(), default_chain: config.default_chain.clone(), @@ -201,6 +208,11 @@ impl EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .into(), + link_to_prover: self + .link_to_prover + .absolutize() + .expect("Failed to parse prover path") + .into(), chains: self.chains.clone(), config: self.config.clone(), default_chain: self.default_chain.clone(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index 4daab36c56b8..af31120b3a49 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -51,7 +51,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { shell.create_dir(ecosystem_name)?; shell.change_dir(ecosystem_name); - let configs_path = create_local_configs_dir(shell, ".")?; + let configs_path: PathBuf = create_local_configs_dir(shell, ".")?; let link_to_code = if args.link_to_code.is_empty() { let spinner = Spinner::new(MSG_CLONING_ERA_REPO_SPINNER); @@ -64,6 +64,9 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { path }; + let mut link_to_prover = link_to_code.clone().into_os_string(); + link_to_prover.push("/prover"); + let spinner = Spinner::new(MSG_CREATING_INITIAL_CONFIGURATIONS_SPINNER); let chain_config = args.chain_config(); let chains_path = shell.create_dir("chains")?; @@ -76,6 +79,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { name: ecosystem_name.clone(), l1_network: args.l1_network, link_to_code: link_to_code.clone(), + link_to_prover: link_to_prover.into(), chains: chains_path.clone(), config: configs_path, era_chain_id: get_default_era_chain_id(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index f82b798b5cba..4804f11da10f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -9,9 +9,8 @@ use crate::messages::{MSG_GENERATING_VK_SPINNER, MSG_INITIALIZING_PROVER, MSG_PR pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { logger::info(MSG_INITIALIZING_PROVER); let ecosystem_config = EcosystemConfig::from_file(shell)?; - let mut link_to_prover = ecosystem_config.link_to_code.into_os_string(); - link_to_prover.push("/prover"); - shell.change_dir(link_to_prover.clone()); + let link_to_prover = ecosystem_config.link_to_prover; + shell.change_dir(&link_to_prover); let spinner = Spinner::new(MSG_GENERATING_VK_SPINNER); let mut cmd = Cmd::new(cmd!( From 8766e45c2ce242ae511269d042c9cf04e030fa8f Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Fri, 14 Jun 2024 13:08:42 +0200 Subject: [PATCH 07/87] Add link_to_prover to ZkStack config --- ZkStack.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ZkStack.yaml b/ZkStack.yaml index 33af09572190..7993598f7802 100644 --- a/ZkStack.yaml +++ b/ZkStack.yaml @@ -1,6 +1,7 @@ name: zk l1_network: Localhost link_to_code: . +link_to_prover: ./prover chains: ./chains config: ./configs/ default_chain: era From 511c318b3942c7b83faacb555c91f0f853f49eef Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Fri, 14 Jun 2024 17:03:28 +0200 Subject: [PATCH 08/87] Remove type annotation --- zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index af31120b3a49..c7ad0ebfe369 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -51,7 +51,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { shell.create_dir(ecosystem_name)?; shell.change_dir(ecosystem_name); - let configs_path: PathBuf = create_local_configs_dir(shell, ".")?; + let configs_path = create_local_configs_dir(shell, ".")?; let link_to_code = if args.link_to_code.is_empty() { let spinner = Spinner::new(MSG_CLONING_ERA_REPO_SPINNER); From f58ef92adf0d7b08cf63c3787e402864b9932812 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 18 Jun 2024 11:11:55 +0200 Subject: [PATCH 09/87] Rename command to generate-sk --- .../zk_inception/src/commands/prover/args/init.rs | 5 ----- .../zk_inception/src/commands/prover/args/mod.rs | 1 - .../src/commands/prover/{init.rs => generate_sk.rs} | 10 ++++------ .../crates/zk_inception/src/commands/prover/mod.rs | 8 +++----- zk_toolbox/crates/zk_inception/src/messages.rs | 5 ++--- 5 files changed, 9 insertions(+), 20 deletions(-) delete mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs delete mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs rename zk_toolbox/crates/zk_inception/src/commands/prover/{init.rs => generate_sk.rs} (64%) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs deleted file mode 100644 index 8a8709b69e25..000000000000 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs +++ /dev/null @@ -1,5 +0,0 @@ -use clap::Parser; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Serialize, Deserialize, Parser)] -pub struct InitArgs {} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs deleted file mode 100644 index 43763f10a418..000000000000 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod init; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs similarity index 64% rename from zk_toolbox/crates/zk_inception/src/commands/prover/init.rs rename to zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs index 4804f11da10f..ce63af761f48 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs @@ -3,16 +3,14 @@ use common::{cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; -use super::args::init::InitArgs; -use crate::messages::{MSG_GENERATING_VK_SPINNER, MSG_INITIALIZING_PROVER, MSG_PROVER_INITIALIZED}; +use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; -pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { - logger::info(MSG_INITIALIZING_PROVER); +pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; let link_to_prover = ecosystem_config.link_to_prover; shell.change_dir(&link_to_prover); - let spinner = Spinner::new(MSG_GENERATING_VK_SPINNER); + let spinner = Spinner::new(MSG_GENERATING_SK_SPINNER); let mut cmd = Cmd::new(cmd!( shell, "cargo run --features gpu --release --bin key_generator -- @@ -22,7 +20,7 @@ pub(crate) async fn run(_args: InitArgs, shell: &Shell) -> anyhow::Result<()> { )); cmd.run()?; spinner.finish(); - logger::outro(MSG_PROVER_INITIALIZED); + logger::outro(MSG_SK_GENERATED); Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index fef963f624eb..ea0843eb0f96 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,17 +1,15 @@ -use args::init::InitArgs; use clap::Subcommand; use xshell::Shell; -mod args; -mod init; +mod generate_sk; #[derive(Subcommand, Debug)] pub enum ProverCommands { /// Initialize prover - Init(InitArgs), + GenerateSK, } pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { match args { - ProverCommands::Init(args) => init::run(args, shell).await, + ProverCommands::GenerateSK => generate_sk::run(shell).await, } } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index c54dbf3995c2..1b3c05258753 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -190,6 +190,5 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( } /// Prover related messages -pub(super) const MSG_INITIALIZING_PROVER: &str = "Initializing prover"; -pub(super) const MSG_PROVER_INITIALIZED: &str = "Prover initialized successfully"; -pub(super) const MSG_GENERATING_VK_SPINNER: &str = "Generating verification keys..."; +pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; +pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; From 3891d63a2c728cfed652f75bf2a9e50ef9853f03 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 18 Jun 2024 11:20:06 +0200 Subject: [PATCH 10/87] Add get_link_to_prover --- zk_toolbox/crates/config/src/ecosystem.rs | 12 ------------ .../zk_inception/src/commands/ecosystem/create.rs | 4 ---- .../zk_inception/src/commands/prover/generate_sk.rs | 4 +++- .../crates/zk_inception/src/commands/prover/mod.rs | 1 + .../crates/zk_inception/src/commands/prover/utils.rs | 10 ++++++++++ 5 files changed, 14 insertions(+), 17 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 932e1177b10f..1557ab21646f 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -25,7 +25,6 @@ struct EcosystemConfigInternal { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, - pub link_to_prover: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -41,7 +40,6 @@ pub struct EcosystemConfig { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, - pub link_to_prover: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -74,11 +72,6 @@ impl<'de> Deserialize<'de> for EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .to_path_buf(), - link_to_prover: config - .link_to_prover - .absolutize() - .expect("Failed to parse prover path") - .to_path_buf(), chains: config.chains.clone(), config: config.config.clone(), default_chain: config.default_chain.clone(), @@ -208,11 +201,6 @@ impl EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .into(), - link_to_prover: self - .link_to_prover - .absolutize() - .expect("Failed to parse prover path") - .into(), chains: self.chains.clone(), config: self.config.clone(), default_chain: self.default_chain.clone(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index c7ad0ebfe369..4daab36c56b8 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -64,9 +64,6 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { path }; - let mut link_to_prover = link_to_code.clone().into_os_string(); - link_to_prover.push("/prover"); - let spinner = Spinner::new(MSG_CREATING_INITIAL_CONFIGURATIONS_SPINNER); let chain_config = args.chain_config(); let chains_path = shell.create_dir("chains")?; @@ -79,7 +76,6 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { name: ecosystem_name.clone(), l1_network: args.l1_network, link_to_code: link_to_code.clone(), - link_to_prover: link_to_prover.into(), chains: chains_path.clone(), config: configs_path, era_chain_id: get_default_era_chain_id(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs index ce63af761f48..a12832923aea 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs @@ -5,9 +5,11 @@ use xshell::{cmd, Shell}; use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; +use super::utils::get_link_to_prover; + pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; - let link_to_prover = ecosystem_config.link_to_prover; + let link_to_prover = get_link_to_prover(&ecosystem_config); shell.change_dir(&link_to_prover); let spinner = Spinner::new(MSG_GENERATING_SK_SPINNER); diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index ea0843eb0f96..c617b915a52c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,6 +1,7 @@ use clap::Subcommand; use xshell::Shell; mod generate_sk; +mod utils; #[derive(Subcommand, Debug)] pub enum ProverCommands { diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs new file mode 100644 index 000000000000..4dae70863dc9 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +use config::EcosystemConfig; + +pub(crate) fn get_link_to_prover(config: &EcosystemConfig) -> PathBuf { + let link_to_code = config.link_to_code.clone(); + let mut link_to_prover = link_to_code.into_os_string(); + link_to_prover.push("/prover"); + link_to_prover.into() +} From 595e98fbfce65b81aebb06f202d24f69f0678d31 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 18 Jun 2024 11:20:59 +0200 Subject: [PATCH 11/87] fmt --- .../crates/zk_inception/src/commands/prover/generate_sk.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs index a12832923aea..a14dd6fb87e5 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs @@ -3,9 +3,8 @@ use common::{cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; -use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; - use super::utils::get_link_to_prover; +use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; From 8a84d66736d82a5e1a4b071f3a0815ea7f93346b Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 18 Jun 2024 14:20:02 +0200 Subject: [PATCH 12/87] Remove link_to_prover from ZkStack config --- ZkStack.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ZkStack.yaml b/ZkStack.yaml index 7993598f7802..33af09572190 100644 --- a/ZkStack.yaml +++ b/ZkStack.yaml @@ -1,7 +1,6 @@ name: zk l1_network: Localhost link_to_code: . -link_to_prover: ./prover chains: ./chains config: ./configs/ default_chain: era From 50f1a1d0d39f0f3986c044d97b481ebdc1ce78d9 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 18 Jun 2024 15:53:10 +0200 Subject: [PATCH 13/87] Add ProverCommands::Run --- .../zk_inception/src/commands/prover/mod.rs | 4 ++++ .../zk_inception/src/commands/prover/run.rs | 15 +++++++++++++++ 2 files changed, 19 insertions(+) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/run.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index c617b915a52c..810a37c01939 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,16 +1,20 @@ use clap::Subcommand; use xshell::Shell; mod generate_sk; +mod run; mod utils; #[derive(Subcommand, Debug)] pub enum ProverCommands { /// Initialize prover GenerateSK, + /// Run prover + Run, } pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { match args { ProverCommands::GenerateSK => generate_sk::run(shell).await, + ProverCommands::Run => run::run(shell).await, } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs new file mode 100644 index 000000000000..0a21ec61edb5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -0,0 +1,15 @@ +use common::logger; +use config::EcosystemConfig; +use xshell::Shell; + +use super::utils::get_link_to_prover; + +pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let link_to_prover = get_link_to_prover(&ecosystem_config); + shell.change_dir(&link_to_prover); + + logger::info("Running prover"); + + Ok(()) +} From 5ab441de59386700eda4fb4793c81cc7780a4234 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 11:43:04 +0200 Subject: [PATCH 14/87] Add ProverRunArgs --- zk_toolbox/Cargo.lock | 3 ++ zk_toolbox/Cargo.toml | 2 +- .../src/commands/prover/args/mod.rs | 1 + .../src/commands/prover/args/run.rs | 53 +++++++++++++++++++ .../zk_inception/src/commands/prover/mod.rs | 6 ++- .../zk_inception/src/commands/prover/run.rs | 6 ++- .../crates/zk_inception/src/messages.rs | 4 ++ 7 files changed, 70 insertions(+), 5 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock index 2827c32a295f..b0d6cab3e1b1 100644 --- a/zk_toolbox/Cargo.lock +++ b/zk_toolbox/Cargo.lock @@ -3530,6 +3530,9 @@ name = "strum" version = "0.26.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d8cec3501a5194c432b2b7976db6b7d10ec95c253208b45f83f7136aa985e29" +dependencies = [ + "strum_macros 0.26.2", +] [[package]] name = "strum_macros" diff --git a/zk_toolbox/Cargo.toml b/zk_toolbox/Cargo.toml index 42ea31c033d3..7b09c955a697 100644 --- a/zk_toolbox/Cargo.toml +++ b/zk_toolbox/Cargo.toml @@ -43,7 +43,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" sqlx = { version = "0.7.4", features = ["runtime-tokio", "migrate", "postgres"] } -strum = "0.26.2" +strum = { workspace = true, features = ["derive"] } strum_macros = "0.26.2" thiserror = "1.0.57" tokio = { version = "1.37", features = ["full"] } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs new file mode 100644 index 000000000000..9137f27b0757 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs @@ -0,0 +1 @@ +pub mod run; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs new file mode 100644 index 000000000000..6327869bceef --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs @@ -0,0 +1,53 @@ +use clap::{Parser, ValueEnum}; +use common::PromptConfirm; +use strum::{EnumIter, IntoEnumIterator}; + +use crate::messages::msg_run_prover_component_prompt; + +#[derive(Debug, Clone, Parser, Default)] +pub struct ProverRunArgs { + #[clap(long, value_delimiter = ',')] + pub components: Option>, +} + +#[derive(Debug, Clone, ValueEnum, strum::EnumString, EnumIter)] +pub enum ProverComponent { + Gateway, + WitnessGenerator, + WitnessVectorGenerator, + Prover, + Compressor, +} + +impl std::fmt::Display for ProverComponent { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProverComponent::Gateway => write!(f, "gateway"), + ProverComponent::WitnessGenerator => write!(f, "witness generator"), + ProverComponent::WitnessVectorGenerator => write!(f, "witness vector generator"), + ProverComponent::Prover => write!(f, "prover"), + ProverComponent::Compressor => write!(f, "compressor"), + } + } +} + +impl ProverRunArgs { + pub fn fill_values_with_prompt(&self) -> anyhow::Result { + let components = self.components.clone().unwrap_or_else(|| { + let mut c = vec![]; + for component in ProverComponent::iter() { + if PromptConfirm::new(msg_run_prover_component_prompt(&component.to_string())) + .default(true) + .ask() + { + c.push(component); + } + } + c + }); + + Ok(ProverRunArgs { + components: Some(components), + }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index 810a37c01939..be24e026fcd2 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,5 +1,7 @@ +use args::run::ProverRunArgs; use clap::Subcommand; use xshell::Shell; +mod args; mod generate_sk; mod run; mod utils; @@ -9,12 +11,12 @@ pub enum ProverCommands { /// Initialize prover GenerateSK, /// Run prover - Run, + Run(ProverRunArgs), } pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { match args { ProverCommands::GenerateSK => generate_sk::run(shell).await, - ProverCommands::Run => run::run(shell).await, + ProverCommands::Run(args) => run::run(args, shell).await, } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 0a21ec61edb5..25fcc6b08ab4 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -2,9 +2,11 @@ use common::logger; use config::EcosystemConfig; use xshell::Shell; -use super::utils::get_link_to_prover; +use super::{args::run::ProverRunArgs, utils::get_link_to_prover}; -pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { +pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { + let args = args.fill_values_with_prompt()?; + logger::debug(format!("Prover args: {:?}", args)); let ecosystem_config = EcosystemConfig::from_file(shell)?; let link_to_prover = get_link_to_prover(&ecosystem_config); shell.change_dir(&link_to_prover); diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 1b3c05258753..c5f05a2485fe 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -192,3 +192,7 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; + +pub fn msg_run_prover_component_prompt(component: &str) -> String { + format!("Do you want to run the {}?", component) +} From 6bb0d45439f7f08258863d036a84f292d8e565fb Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 11:52:13 +0200 Subject: [PATCH 15/87] Add run functions --- .../zk_inception/src/commands/prover/run.rs | 46 ++++++++++++++++++- .../crates/zk_inception/src/messages.rs | 6 +++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 25fcc6b08ab4..d93f70ddc56d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -2,7 +2,16 @@ use common::logger; use config::EcosystemConfig; use xshell::Shell; -use super::{args::run::ProverRunArgs, utils::get_link_to_prover}; +use crate::messages::{ + MSG_MISSING_COMPONENTS_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, + MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_WITNESS_GENERATOR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR, +}; + +use super::{ + args::run::{ProverComponent, ProverRunArgs}, + utils::get_link_to_prover, +}; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { let args = args.fill_values_with_prompt()?; @@ -11,7 +20,40 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() let link_to_prover = get_link_to_prover(&ecosystem_config); shell.change_dir(&link_to_prover); - logger::info("Running prover"); + for component in args.components.expect(MSG_MISSING_COMPONENTS_ERR) { + match component { + ProverComponent::Gateway => run_gateway(shell)?, + ProverComponent::WitnessGenerator => run_witness_generator(shell)?, + ProverComponent::WitnessVectorGenerator => run_witness_vector_generator(shell)?, + ProverComponent::Prover => run_prover(shell)?, + ProverComponent::Compressor => run_compressor(shell)?, + } + } Ok(()) } + +fn run_gateway(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RUNNING_PROVER_GATEWAY); + todo!() +} + +fn run_witness_generator(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RUNNING_WITNESS_GENERATOR); + todo!() +} + +fn run_witness_vector_generator(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RUNNING_WITNESS_VECTOR_GENERATOR); + todo!() +} + +fn run_prover(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RUNNING_PROVER); + todo!() +} + +fn run_compressor(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RUNNING_COMPRESSOR); + todo!() +} diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index c5f05a2485fe..0c85f455adbe 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -192,6 +192,12 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; +pub(super) const MSG_MISSING_COMPONENTS_ERR: &str = "Missing components"; +pub(super) const MSG_RUNNING_PROVER_GATEWAY: &str = "Running gateway"; +pub(super) const MSG_RUNNING_WITNESS_GENERATOR: &str = "Running witness generator"; +pub(super) const MSG_RUNNING_WITNESS_VECTOR_GENERATOR: &str = "Running witness vector generator"; +pub(super) const MSG_RUNNING_PROVER: &str = "Running prover"; +pub(super) const MSG_RUNNING_COMPRESSOR: &str = "Running compressor"; pub fn msg_run_prover_component_prompt(component: &str) -> String { format!("Do you want to run the {}?", component) From c6c6fa83fce35d9055de0c898b4bd456068a76bb Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 11:52:34 +0200 Subject: [PATCH 16/87] fmt --- .../crates/zk_inception/src/commands/prover/run.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index d93f70ddc56d..5b4bd065011e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -2,17 +2,16 @@ use common::logger; use config::EcosystemConfig; use xshell::Shell; +use super::{ + args::run::{ProverComponent, ProverRunArgs}, + utils::get_link_to_prover, +}; use crate::messages::{ MSG_MISSING_COMPONENTS_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, }; -use super::{ - args::run::{ProverComponent, ProverRunArgs}, - utils::get_link_to_prover, -}; - pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { let args = args.fill_values_with_prompt()?; logger::debug(format!("Prover args: {:?}", args)); From 515f3a9378ced7fba15d624f73bec0190c7b123f Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 17:46:15 +0200 Subject: [PATCH 17/87] Add run_gateway --- zk_toolbox/crates/config/src/chain.rs | 11 +++++++- .../zk_inception/src/commands/prover/run.rs | 25 +++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/zk_toolbox/crates/config/src/chain.rs b/zk_toolbox/crates/config/src/chain.rs index e685b0966b44..e84eae3204fe 100644 --- a/zk_toolbox/crates/config/src/chain.rs +++ b/zk_toolbox/crates/config/src/chain.rs @@ -11,7 +11,8 @@ use xshell::Shell; use crate::{ consts::{ - CONFIG_NAME, CONTRACTS_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, SECRETS_FILE, WALLETS_FILE, + CONFIG_NAME, CONTRACTS_FILE, GENERAL_FILE, GENESIS_FILE, L1_CONTRACTS_FOUNDRY, + SECRETS_FILE, WALLETS_FILE, }, create_localhost_wallets, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, SaveConfigWithBasePath}, @@ -91,6 +92,14 @@ impl ChainConfig { SecretsConfig::read(self.get_shell(), self.configs.join(SECRETS_FILE)) } + pub fn path_to_general_config(&self) -> PathBuf { + self.configs.join(GENERAL_FILE) + } + + pub fn path_to_secrets_config(&self) -> PathBuf { + self.configs.join(SECRETS_FILE) + } + pub fn path_to_foundry(&self) -> PathBuf { self.link_to_code.join(L1_CONTRACTS_FOUNDRY) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 5b4bd065011e..33623e3f6cbc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,14 +1,16 @@ -use common::logger; -use config::EcosystemConfig; -use xshell::Shell; +use std::path::PathBuf; + +use common::{cmd::Cmd, logger}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::{cmd, Shell}; use super::{ args::run::{ProverComponent, ProverRunArgs}, utils::get_link_to_prover, }; use crate::messages::{ - MSG_MISSING_COMPONENTS_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, - MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_WITNESS_GENERATOR, + MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENTS_ERR, MSG_RUNNING_COMPRESSOR, + MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, }; @@ -16,12 +18,16 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() let args = args.fill_values_with_prompt()?; logger::debug(format!("Prover args: {:?}", args)); let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain = ecosystem_config + .load_chain(Some(ecosystem_config.default_chain.clone())) + .expect(MSG_CHAIN_NOT_FOUND_ERR); + let link_to_prover = get_link_to_prover(&ecosystem_config); shell.change_dir(&link_to_prover); for component in args.components.expect(MSG_MISSING_COMPONENTS_ERR) { match component { - ProverComponent::Gateway => run_gateway(shell)?, + ProverComponent::Gateway => run_gateway(shell, &chain)?, ProverComponent::WitnessGenerator => run_witness_generator(shell)?, ProverComponent::WitnessVectorGenerator => run_witness_vector_generator(shell)?, ProverComponent::Prover => run_prover(shell)?, @@ -32,9 +38,12 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() Ok(()) } -fn run_gateway(shell: &Shell) -> anyhow::Result<()> { +fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); - todo!() + let config_path = chain.path_to_general_config(); + let secrets_path = chain.path_to_secrets_config(); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path}")); + cmd.run() } fn run_witness_generator(shell: &Shell) -> anyhow::Result<()> { From 1b283ab0e21437d954ff4b1c92cd8363eed22a9f Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 17:52:13 +0200 Subject: [PATCH 18/87] Add run in background --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 33623e3f6cbc..60454bd1a74f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -42,7 +42,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path}")); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} &")).with_force_run(); cmd.run() } From a5b663bef3a164f453604f731df6486e93ddcc02 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 17:55:26 +0200 Subject: [PATCH 19/87] fix run in background --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 60454bd1a74f..c08d94ec9174 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -42,7 +42,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} &")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway & -- --config-path={config_path} --secrets-path={secrets_path}")).with_force_run(); cmd.run() } From 627f39ab74c28d32590d93d23b90db8d7e09641d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 17:59:05 +0200 Subject: [PATCH 20/87] Use nohup --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index c08d94ec9174..d826fccf8bab 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -42,7 +42,8 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway & -- --config-path={config_path} --secrets-path={secrets_path}")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} > /dev/null 2>&1 & + ")).with_force_run(); cmd.run() } From 6f060daa4444d5ac624afa9038ff33155b570313 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 17:59:19 +0200 Subject: [PATCH 21/87] fix fmt --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index d826fccf8bab..d3a2b6b5447e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -42,8 +42,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} > /dev/null 2>&1 & - ")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} > /dev/null 2>&1 &")).with_force_run(); cmd.run() } From e9b4ac3428ded40da48a7cd6427cceb9e266b91c Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:15:11 +0200 Subject: [PATCH 22/87] Add PID message --- .../crates/zk_inception/src/commands/prover/run.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index d3a2b6b5447e..1af145393652 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -35,6 +35,8 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() } } + logger::outro("Prover components started successfully".to_string()); + Ok(()) } @@ -42,8 +44,11 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} > /dev/null 2>&1 &")).with_force_run(); - cmd.run() + let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} -- > /dev/null 2>&1 &")).with_force_run(); + let out = String::from_utf8(cmd.run_with_output()?.stdout)?; + let pid = out.split(" ").next().expect("Failed to get pid"); + logger::info(format!("Prover gateway started with pid: {}", pid)); + Ok(()) } fn run_witness_generator(shell: &Shell) -> anyhow::Result<()> { From bccf534374972a731d4ab6ce67f679f1e96de529 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:28:55 +0200 Subject: [PATCH 23/87] add sh --- .../crates/zk_inception/src/commands/prover/run.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 1af145393652..fecd6f46c8ff 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -44,7 +44,14 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "nohup cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} -- > /dev/null 2>&1 &")).with_force_run(); + let command = + format!( + "\'cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}\'", + config_path.to_str().expect("Failed to convert path to string"), + secrets_path.to_str().expect("Failed to convert path to string") + ); + let mut cmd = + Cmd::new(cmd!(shell, "nohup sh -c {command} > /dev/null 2>&1 &")).with_force_run(); let out = String::from_utf8(cmd.run_with_output()?.stdout)?; let pid = out.split(" ").next().expect("Failed to get pid"); logger::info(format!("Prover gateway started with pid: {}", pid)); From ffe590157756009089a42b2a4174d9fe7a9be25d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:30:25 +0200 Subject: [PATCH 24/87] remove \' --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index fecd6f46c8ff..2a51e0cf7970 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -46,7 +46,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let secrets_path = chain.path_to_secrets_config(); let command = format!( - "\'cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}\'", + "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", config_path.to_str().expect("Failed to convert path to string"), secrets_path.to_str().expect("Failed to convert path to string") ); From 193ff582dfb753a395400d755a25a6c41d9230fb Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:32:18 +0200 Subject: [PATCH 25/87] remove sh --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 2a51e0cf7970..82c8456752fd 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -50,8 +50,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { config_path.to_str().expect("Failed to convert path to string"), secrets_path.to_str().expect("Failed to convert path to string") ); - let mut cmd = - Cmd::new(cmd!(shell, "nohup sh -c {command} > /dev/null 2>&1 &")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "nohup {command} > /dev/null 2>&1 &")).with_force_run(); let out = String::from_utf8(cmd.run_with_output()?.stdout)?; let pid = out.split(" ").next().expect("Failed to get pid"); logger::info(format!("Prover gateway started with pid: {}", pid)); From aee700e24a4ba5439bc09832de22c73123063a38 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:34:51 +0200 Subject: [PATCH 26/87] Remove nohup --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 82c8456752fd..593c68f1dbee 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -50,7 +50,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { config_path.to_str().expect("Failed to convert path to string"), secrets_path.to_str().expect("Failed to convert path to string") ); - let mut cmd = Cmd::new(cmd!(shell, "nohup {command} > /dev/null 2>&1 &")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "{command} > /dev/null 2>&1 &")).with_force_run(); let out = String::from_utf8(cmd.run_with_output()?.stdout)?; let pid = out.split(" ").next().expect("Failed to get pid"); logger::info(format!("Prover gateway started with pid: {}", pid)); From b620f257f4da20b1d2b6f433ec35338e2ae5c19b Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 25 Jun 2024 18:36:54 +0200 Subject: [PATCH 27/87] remove command variable --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 593c68f1dbee..dc1e8bec4f18 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -44,13 +44,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let command = - format!( - "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", - config_path.to_str().expect("Failed to convert path to string"), - secrets_path.to_str().expect("Failed to convert path to string") - ); - let mut cmd = Cmd::new(cmd!(shell, "{command} > /dev/null 2>&1 &")).with_force_run(); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} -- > /dev/null 2>&1 &")).with_force_run(); let out = String::from_utf8(cmd.run_with_output()?.stdout)?; let pid = out.split(" ").next().expect("Failed to get pid"); logger::info(format!("Prover gateway started with pid: {}", pid)); From fe6aabe33db14fd8e4b0c7e5361d85c8db51d34b Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 26 Jun 2024 14:17:03 +0200 Subject: [PATCH 28/87] Run with command --- .../zk_inception/src/commands/prover/run.rs | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index dc1e8bec4f18..e2ebf28d268f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,4 +1,7 @@ -use std::path::PathBuf; +use std::{ + path::PathBuf, + process::{Command, Stdio}, +}; use common::{cmd::Cmd, logger}; use config::{ChainConfig, EcosystemConfig}; @@ -44,9 +47,23 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path} -- > /dev/null 2>&1 &")).with_force_run(); - let out = String::from_utf8(cmd.run_with_output()?.stdout)?; - let pid = out.split(" ").next().expect("Failed to get pid"); + + let command_str = format!( + "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", + config_path.to_str().unwrap(), + secrets_path.to_str().unwrap() + ); + + let cmd = Command::new("sh") + .arg("-c") + .arg(&command_str) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn()?; + + // Get the PID of the spawned process + let pid = cmd.id(); + logger::info(format!("Prover gateway started with pid: {}", pid)); Ok(()) } From 364664cd996cbcc91d923678b9cdd0d516474408 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 26 Jun 2024 14:19:55 +0200 Subject: [PATCH 29/87] Update current dur --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index e2ebf28d268f..bb40fbe8d2d1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -47,6 +47,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); + let link_to_prover = shell.current_dir(); let command_str = format!( "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", @@ -55,10 +56,9 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { ); let cmd = Command::new("sh") + .current_dir(link_to_prover) .arg("-c") .arg(&command_str) - .stdout(Stdio::null()) - .stderr(Stdio::null()) .spawn()?; // Get the PID of the spawned process From ccf013c9f2a1e2735058bb1093ea3552e28d0f2c Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 26 Jun 2024 14:35:48 +0200 Subject: [PATCH 30/87] Refactor msgs --- zk_toolbox/crates/common/src/cmd.rs | 17 ++++++++- .../zk_inception/src/commands/prover/run.rs | 36 +++++++++---------- .../crates/zk_inception/src/messages.rs | 4 +++ 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/zk_toolbox/crates/common/src/cmd.rs b/zk_toolbox/crates/common/src/cmd.rs index e39f1e18972c..d58b4fc770ee 100644 --- a/zk_toolbox/crates/common/src/cmd.rs +++ b/zk_toolbox/crates/common/src/cmd.rs @@ -1,4 +1,7 @@ -use std::process::Output; +use std::{ + path::PathBuf, + process::{Command, Output, Stdio}, +}; use anyhow::bail; use console::style; @@ -147,3 +150,15 @@ fn get_indented_output( indent(&wrap_text_to_len(&stderr)), ) } + +pub fn run_in_background(command: &str, wd: &PathBuf) -> anyhow::Result { + let cmd = Command::new("sh") + .current_dir(wd) + .arg("-c") + .arg(command) + .stdout(Stdio::inherit()) + .stderr(Stdio::inherit()) + .spawn()?; + + Ok(cmd.id()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index bb40fbe8d2d1..db187ef536dc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -3,7 +3,10 @@ use std::{ process::{Command, Stdio}, }; -use common::{cmd::Cmd, logger}; +use common::{ + cmd::{run_in_background, Cmd}, + logger, +}; use config::{ChainConfig, EcosystemConfig}; use xshell::{cmd, Shell}; @@ -12,9 +15,9 @@ use super::{ utils::get_link_to_prover, }; use crate::messages::{ - MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENTS_ERR, MSG_RUNNING_COMPRESSOR, - MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_WITNESS_GENERATOR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + msg_started_component, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENTS_ERR, + MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, + MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -43,11 +46,10 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() Ok(()) } -fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { +fn run_gateway(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let link_to_prover = shell.current_dir(); let command_str = format!( "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", @@ -55,35 +57,31 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { secrets_path.to_str().unwrap() ); - let cmd = Command::new("sh") - .current_dir(link_to_prover) - .arg("-c") - .arg(&command_str) - .spawn()?; - - // Get the PID of the spawned process - let pid = cmd.id(); + let pid = run_in_background(&command_str, link_to_prover)?; - logger::info(format!("Prover gateway started with pid: {}", pid)); + logger::info(msg_started_component("Prover gateway", pid)); Ok(()) } -fn run_witness_generator(shell: &Shell) -> anyhow::Result<()> { +fn run_witness_generator(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_GENERATOR); todo!() } -fn run_witness_vector_generator(shell: &Shell) -> anyhow::Result<()> { +fn run_witness_vector_generator( + chain: &ChainConfig, + link_to_prover: &PathBuf, +) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_VECTOR_GENERATOR); todo!() } -fn run_prover(shell: &Shell) -> anyhow::Result<()> { +fn run_prover(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER); todo!() } -fn run_compressor(shell: &Shell) -> anyhow::Result<()> { +fn run_compressor(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { logger::info(MSG_RUNNING_COMPRESSOR); todo!() } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 0c85f455adbe..9640083be791 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -199,6 +199,10 @@ pub(super) const MSG_RUNNING_WITNESS_VECTOR_GENERATOR: &str = "Running witness v pub(super) const MSG_RUNNING_PROVER: &str = "Running prover"; pub(super) const MSG_RUNNING_COMPRESSOR: &str = "Running compressor"; +pub fn msg_started_component(component: &str, pid: u32) -> String { + format!("{} started with pid {}", component, pid) +} + pub fn msg_run_prover_component_prompt(component: &str) -> String { format!("Do you want to run the {}?", component) } From f5dea76eb3bfa18a5d594b9c39062c84d957c2d8 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 26 Jun 2024 14:36:35 +0200 Subject: [PATCH 31/87] Fix params --- .../crates/zk_inception/src/commands/prover/run.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index db187ef536dc..12cc415d386f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -29,15 +29,16 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() .expect(MSG_CHAIN_NOT_FOUND_ERR); let link_to_prover = get_link_to_prover(&ecosystem_config); - shell.change_dir(&link_to_prover); for component in args.components.expect(MSG_MISSING_COMPONENTS_ERR) { match component { - ProverComponent::Gateway => run_gateway(shell, &chain)?, - ProverComponent::WitnessGenerator => run_witness_generator(shell)?, - ProverComponent::WitnessVectorGenerator => run_witness_vector_generator(shell)?, - ProverComponent::Prover => run_prover(shell)?, - ProverComponent::Compressor => run_compressor(shell)?, + ProverComponent::Gateway => run_gateway(&chain, &link_to_prover)?, + ProverComponent::WitnessGenerator => run_witness_generator(&chain, &link_to_prover)?, + ProverComponent::WitnessVectorGenerator => { + run_witness_vector_generator(&chain, &link_to_prover)? + } + ProverComponent::Prover => run_prover(&chain, &link_to_prover)?, + ProverComponent::Compressor => run_compressor(&chain, &link_to_prover)?, } } From 65e6399aad921fd3a2525c9351c8679cf5276d72 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 11:31:29 +0200 Subject: [PATCH 32/87] Remove run in background --- .../src/commands/prover/args/run.rs | 25 +++----- .../zk_inception/src/commands/prover/run.rs | 62 +++++++------------ .../crates/zk_inception/src/messages.rs | 12 +--- 3 files changed, 33 insertions(+), 66 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs index 6327869bceef..750394bbe402 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs @@ -1,16 +1,16 @@ use clap::{Parser, ValueEnum}; -use common::PromptConfirm; +use common::PromptSelect; use strum::{EnumIter, IntoEnumIterator}; -use crate::messages::msg_run_prover_component_prompt; +use crate::messages::MSG_RUN_COMPONENT_PROMPT; #[derive(Debug, Clone, Parser, Default)] pub struct ProverRunArgs { - #[clap(long, value_delimiter = ',')] - pub components: Option>, + #[clap(long)] + pub component: Option, } -#[derive(Debug, Clone, ValueEnum, strum::EnumString, EnumIter)] +#[derive(Debug, Clone, ValueEnum, strum::EnumString, EnumIter, PartialEq, Eq)] pub enum ProverComponent { Gateway, WitnessGenerator, @@ -33,21 +33,12 @@ impl std::fmt::Display for ProverComponent { impl ProverRunArgs { pub fn fill_values_with_prompt(&self) -> anyhow::Result { - let components = self.components.clone().unwrap_or_else(|| { - let mut c = vec![]; - for component in ProverComponent::iter() { - if PromptConfirm::new(msg_run_prover_component_prompt(&component.to_string())) - .default(true) - .ask() - { - c.push(component); - } - } - c + let component = self.component.clone().unwrap_or_else(|| { + PromptSelect::new(MSG_RUN_COMPONENT_PROMPT, ProverComponent::iter()).ask() }); Ok(ProverRunArgs { - components: Some(components), + component: Some(component), }) } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 12cc415d386f..d75c7df1c47f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,12 +1,5 @@ -use std::{ - path::PathBuf, - process::{Command, Stdio}, -}; - -use common::{ - cmd::{run_in_background, Cmd}, - logger, -}; +use anyhow::Context; +use common::{cmd::Cmd, logger}; use config::{ChainConfig, EcosystemConfig}; use xshell::{cmd, Shell}; @@ -15,74 +8,63 @@ use super::{ utils::get_link_to_prover, }; use crate::messages::{ - msg_started_component, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENTS_ERR, - MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, - MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, + MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { let args = args.fill_values_with_prompt()?; - logger::debug(format!("Prover args: {:?}", args)); let ecosystem_config = EcosystemConfig::from_file(shell)?; let chain = ecosystem_config .load_chain(Some(ecosystem_config.default_chain.clone())) .expect(MSG_CHAIN_NOT_FOUND_ERR); let link_to_prover = get_link_to_prover(&ecosystem_config); + shell.change_dir(link_to_prover.clone()); - for component in args.components.expect(MSG_MISSING_COMPONENTS_ERR) { - match component { - ProverComponent::Gateway => run_gateway(&chain, &link_to_prover)?, - ProverComponent::WitnessGenerator => run_witness_generator(&chain, &link_to_prover)?, - ProverComponent::WitnessVectorGenerator => { - run_witness_vector_generator(&chain, &link_to_prover)? - } - ProverComponent::Prover => run_prover(&chain, &link_to_prover)?, - ProverComponent::Compressor => run_compressor(&chain, &link_to_prover)?, + match args.component { + Some(ProverComponent::Gateway) => run_gateway(shell, &chain)?, + Some(ProverComponent::WitnessGenerator) => run_witness_generator(shell, &chain)?, + Some(ProverComponent::WitnessVectorGenerator) => { + run_witness_vector_generator(shell, &chain)? } + Some(ProverComponent::Prover) => run_prover(shell, &chain)?, + Some(ProverComponent::Compressor) => run_compressor(shell, &chain)?, + None => anyhow::bail!(MSG_MISSING_COMPONENT_ERR), } - logger::outro("Prover components started successfully".to_string()); - Ok(()) } -fn run_gateway(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { +fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let command_str = format!( - "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={} --secrets-path={}", - config_path.to_str().unwrap(), - secrets_path.to_str().unwrap() - ); - - let pid = run_in_background(&command_str, link_to_prover)?; + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path}")); + cmd = cmd.with_force_run(); + cmd.run().context(MSG_RUNNING_PROVER_GATEWAY_ERR)?; - logger::info(msg_started_component("Prover gateway", pid)); Ok(()) } -fn run_witness_generator(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { +fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_GENERATOR); todo!() } -fn run_witness_vector_generator( - chain: &ChainConfig, - link_to_prover: &PathBuf, -) -> anyhow::Result<()> { +fn run_witness_vector_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_VECTOR_GENERATOR); todo!() } -fn run_prover(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { +fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER); todo!() } -fn run_compressor(chain: &ChainConfig, link_to_prover: &PathBuf) -> anyhow::Result<()> { +fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_COMPRESSOR); todo!() } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 9640083be791..88b3e6358f33 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -192,17 +192,11 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( /// Prover related messages pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; -pub(super) const MSG_MISSING_COMPONENTS_ERR: &str = "Missing components"; +pub(super) const MSG_MISSING_COMPONENT_ERR: &str = "Missing component"; pub(super) const MSG_RUNNING_PROVER_GATEWAY: &str = "Running gateway"; pub(super) const MSG_RUNNING_WITNESS_GENERATOR: &str = "Running witness generator"; pub(super) const MSG_RUNNING_WITNESS_VECTOR_GENERATOR: &str = "Running witness vector generator"; pub(super) const MSG_RUNNING_PROVER: &str = "Running prover"; pub(super) const MSG_RUNNING_COMPRESSOR: &str = "Running compressor"; - -pub fn msg_started_component(component: &str, pid: u32) -> String { - format!("{} started with pid {}", component, pid) -} - -pub fn msg_run_prover_component_prompt(component: &str) -> String { - format!("Do you want to run the {}?", component) -} +pub(super) const MSG_RUN_COMPONENT_PROMPT: &str = "What component do you want to run?"; +pub(super) const MSG_RUNNING_PROVER_GATEWAY_ERR: &str = "Failed to run prover gateway"; From f18c527b7279df6d61c5a8f7291cd5ec2b108989 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 11:54:36 +0200 Subject: [PATCH 33/87] Add run_witness_generator --- .../crates/zk_inception/src/commands/prover/run.rs | 11 +++++++++-- zk_toolbox/crates/zk_inception/src/messages.rs | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index d75c7df1c47f..445d42e4256c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -10,7 +10,7 @@ use super::{ use crate::messages::{ MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -51,7 +51,14 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_GENERATOR); - todo!() + let config_path = chain.path_to_general_config(); + let secrets_path = chain.path_to_secrets_config(); + + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- --all_rounds -- --config-path={config_path} --secrets-path={secrets_path}")); + cmd = cmd.with_force_run(); + cmd.run().context(MSG_RUNNING_WITNESS_GENERATOR_ERR)?; + + Ok(()) } fn run_witness_vector_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 88b3e6358f33..5bf26ac9d4d0 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -200,3 +200,4 @@ pub(super) const MSG_RUNNING_PROVER: &str = "Running prover"; pub(super) const MSG_RUNNING_COMPRESSOR: &str = "Running compressor"; pub(super) const MSG_RUN_COMPONENT_PROMPT: &str = "What component do you want to run?"; pub(super) const MSG_RUNNING_PROVER_GATEWAY_ERR: &str = "Failed to run prover gateway"; +pub(super) const MSG_RUNNING_WITNESS_GENERATOR_ERR: &str = "Failed to run witness generator"; From 47236a67a090bf90fac4a7dc2d6375e31e6aff0c Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 12:06:47 +0200 Subject: [PATCH 34/87] Fix run_witness_generator --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- zk_toolbox/crates/zk_inception/src/messages.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 445d42e4256c..7321725c31f6 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -54,7 +54,7 @@ fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<( let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- --all_rounds -- --config-path={config_path} --secrets-path={secrets_path}")); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- --all_rounds --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_WITNESS_GENERATOR_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 5bf26ac9d4d0..7cdf0b8c0092 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -201,3 +201,7 @@ pub(super) const MSG_RUNNING_COMPRESSOR: &str = "Running compressor"; pub(super) const MSG_RUN_COMPONENT_PROMPT: &str = "What component do you want to run?"; pub(super) const MSG_RUNNING_PROVER_GATEWAY_ERR: &str = "Failed to run prover gateway"; pub(super) const MSG_RUNNING_WITNESS_GENERATOR_ERR: &str = "Failed to run witness generator"; +pub(super) const MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR: &str = + "Failed to run witness vector generator"; +pub(super) const MSG_RUNNING_COMPRESSOR_ERR: &str = "Failed to run compressor"; +pub(super) const MSG_RUNNING_PROVER_ERR: &str = "Failed to run prover"; From dca0c41865339f2fab0d59a94d5af397370ca571 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 17:12:22 +0200 Subject: [PATCH 35/87] Add run_witness_vector_generator --- .../zk_inception/src/commands/prover/run.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 7321725c31f6..352f212126bc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -11,6 +11,7 @@ use crate::messages::{ MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -44,9 +45,7 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_prover_fri_gateway -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); - cmd.run().context(MSG_RUNNING_PROVER_GATEWAY_ERR)?; - - Ok(()) + cmd.run().context(MSG_RUNNING_PROVER_GATEWAY_ERR) } fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { @@ -56,14 +55,16 @@ fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<( let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- --all_rounds --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); - cmd.run().context(MSG_RUNNING_WITNESS_GENERATOR_ERR)?; - - Ok(()) + cmd.run().context(MSG_RUNNING_WITNESS_GENERATOR_ERR) } fn run_witness_vector_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_VECTOR_GENERATOR); - todo!() + let config_path = chain.path_to_general_config(); + let secrets_path = chain.path_to_secrets_config(); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_vector_generator -- --config-path={config_path} --secrets-path={secrets_path}")); + cmd = cmd.with_force_run(); + cmd.run().context(MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR) } fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { From c28ca67ac4ecd0a8e2c9c8d7dad2dd481f784ba7 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 17:46:39 +0200 Subject: [PATCH 36/87] Add run_prover --- .../zk_inception/src/commands/prover/run.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 352f212126bc..4283ba16125c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -9,9 +9,9 @@ use super::{ }; use crate::messages::{ MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, - MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, - MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, + MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, + MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -62,6 +62,7 @@ fn run_witness_vector_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::R logger::info(MSG_RUNNING_WITNESS_VECTOR_GENERATOR); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_vector_generator -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR) @@ -69,7 +70,14 @@ fn run_witness_vector_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::R fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_PROVER); - todo!() + let config_path = chain.path_to_general_config(); + let secrets_path = chain.path_to_secrets_config(); + + let mut cmd = Cmd::new( + cmd!(shell, "cargo run --features \"gpu\" --release --bin zksync_prover_fri -- --config-path={config_path} --secrets-path={secrets_path}"), + ); + cmd = cmd.with_force_run(); + cmd.run().context(MSG_RUNNING_PROVER_ERR) } fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { From f2f577b1f73823a94bb4ae8a709df12d36a578e8 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 27 Jun 2024 17:47:52 +0200 Subject: [PATCH 37/87] Fix prover command --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 4283ba16125c..c0af673f82d7 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -74,7 +74,7 @@ fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let secrets_path = chain.path_to_secrets_config(); let mut cmd = Cmd::new( - cmd!(shell, "cargo run --features \"gpu\" --release --bin zksync_prover_fri -- --config-path={config_path} --secrets-path={secrets_path}"), + cmd!(shell, "cargo run --features gpu --release --bin zksync_prover_fri -- --config-path={config_path} --secrets-path={secrets_path}"), ); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_PROVER_ERR) From a9d15800b6c96a829cbd2198bb0cb65254566bc3 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 10:32:54 +0200 Subject: [PATCH 38/87] fmt --- zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index 593514f502c3..06b7dfac80b3 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,13 +1,12 @@ -use args::run::ProverRunArgs; -use args::init::ProverInitArgs; +use args::{init::ProverInitArgs, run::ProverRunArgs}; use clap::Subcommand; use xshell::Shell; mod args; +mod gcs; mod generate_sk; -mod run; mod init; +mod run; mod utils; -mod gcs; #[derive(Subcommand, Debug)] pub enum ProverCommands { From 017850eb1d83e608e249a7eddf971b02bfd4ebc9 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 12:21:35 +0200 Subject: [PATCH 39/87] Add WitnessGeneratorArgs --- .../src/commands/prover/args/run.rs | 67 +++++++++++++++---- .../zk_inception/src/commands/prover/run.rs | 25 +++++-- .../crates/zk_inception/src/messages.rs | 2 + 3 files changed, 78 insertions(+), 16 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs index 750394bbe402..28d5f1bec038 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs @@ -2,33 +2,54 @@ use clap::{Parser, ValueEnum}; use common::PromptSelect; use strum::{EnumIter, IntoEnumIterator}; -use crate::messages::MSG_RUN_COMPONENT_PROMPT; +use crate::messages::{MSG_ROUND_SELECT_PROMPT, MSG_RUN_COMPONENT_PROMPT}; #[derive(Debug, Clone, Parser, Default)] pub struct ProverRunArgs { #[clap(long)] pub component: Option, + #[clap(flatten)] + pub witness_generator_args: WitnessGeneratorArgs, } -#[derive(Debug, Clone, ValueEnum, strum::EnumString, EnumIter, PartialEq, Eq)] +#[derive( + Debug, Clone, ValueEnum, strum::EnumString, EnumIter, PartialEq, Eq, Copy, strum_macros::Display, +)] pub enum ProverComponent { + #[strum(to_string = "Gateway")] Gateway, + #[strum(to_string = "Witness generator")] WitnessGenerator, + #[strum(to_string = "Witness vector generator")] WitnessVectorGenerator, + #[strum(to_string = "Prover")] Prover, + #[strum(to_string = "Compressor")] Compressor, } -impl std::fmt::Display for ProverComponent { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ProverComponent::Gateway => write!(f, "gateway"), - ProverComponent::WitnessGenerator => write!(f, "witness generator"), - ProverComponent::WitnessVectorGenerator => write!(f, "witness vector generator"), - ProverComponent::Prover => write!(f, "prover"), - ProverComponent::Compressor => write!(f, "compressor"), - } - } +#[derive(Debug, Clone, Parser, Default)] +pub struct WitnessGeneratorArgs { + #[clap(long)] + pub round: Option, +} + +#[derive( + Debug, Clone, ValueEnum, strum::EnumString, EnumIter, PartialEq, Eq, strum_macros::Display, +)] +pub enum WitnessGeneratorRound { + #[strum(to_string = "All rounds")] + AllRounds, + #[strum(to_string = "Basic circuits")] + BasicCircuits, + #[strum(to_string = "Leaf aggregation")] + LeafAggregation, + #[strum(to_string = "Node aggregation")] + NodeAggregation, + #[strum(to_string = "Recursion tip")] + RecursionTip, + #[strum(to_string = "Scheduler")] + Scheduler, } impl ProverRunArgs { @@ -37,8 +58,30 @@ impl ProverRunArgs { PromptSelect::new(MSG_RUN_COMPONENT_PROMPT, ProverComponent::iter()).ask() }); + let witness_generator_args = self + .witness_generator_args + .fill_values_with_prompt(component)?; + Ok(ProverRunArgs { component: Some(component), + witness_generator_args, }) } } + +impl WitnessGeneratorArgs { + pub fn fill_values_with_prompt( + &self, + component: ProverComponent, + ) -> anyhow::Result { + if component != ProverComponent::WitnessGenerator { + return Ok(Self::default()); + } + + let round = self.round.clone().unwrap_or_else(|| { + PromptSelect::new(MSG_ROUND_SELECT_PROMPT, WitnessGeneratorRound::iter()).ask() + }); + + Ok(WitnessGeneratorArgs { round: Some(round) }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index c0af673f82d7..975fe3d500f6 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -4,7 +4,7 @@ use config::{ChainConfig, EcosystemConfig}; use xshell::{cmd, Shell}; use super::{ - args::run::{ProverComponent, ProverRunArgs}, + args::run::{ProverComponent, ProverRunArgs, WitnessGeneratorArgs, WitnessGeneratorRound}, utils::get_link_to_prover, }; use crate::messages::{ @@ -12,6 +12,7 @@ use crate::messages::{ MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, + MSG_WITNESS_GENERATOR_ROUND_ERR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -26,7 +27,9 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() match args.component { Some(ProverComponent::Gateway) => run_gateway(shell, &chain)?, - Some(ProverComponent::WitnessGenerator) => run_witness_generator(shell, &chain)?, + Some(ProverComponent::WitnessGenerator) => { + run_witness_generator(shell, &chain, args.witness_generator_args)? + } Some(ProverComponent::WitnessVectorGenerator) => { run_witness_vector_generator(shell, &chain)? } @@ -48,12 +51,26 @@ fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { cmd.run().context(MSG_RUNNING_PROVER_GATEWAY_ERR) } -fn run_witness_generator(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { +fn run_witness_generator( + shell: &Shell, + chain: &ChainConfig, + args: WitnessGeneratorArgs, +) -> anyhow::Result<()> { logger::info(MSG_RUNNING_WITNESS_GENERATOR); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); + let round = args.round.expect(MSG_WITNESS_GENERATOR_ROUND_ERR); + + let round_str = match round { + WitnessGeneratorRound::AllRounds => "--all_rounds", + WitnessGeneratorRound::BasicCircuits => "--round=basic_circuits", + WitnessGeneratorRound::LeafAggregation => "--round=leaf_aggregation", + WitnessGeneratorRound::NodeAggregation => "--round=node_aggregation", + WitnessGeneratorRound::RecursionTip => "--round=recursion_tip", + WitnessGeneratorRound::Scheduler => "--round=scheduler", + }; - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- --all_rounds --config-path={config_path} --secrets-path={secrets_path}")); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin zksync_witness_generator -- {round_str} --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_WITNESS_GENERATOR_ERR) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index df2984fc95a8..a25f3fc9baa5 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -257,6 +257,8 @@ pub(super) const MSG_GETTING_PROOF_STORE_CONFIG: &str = "Getting proof store con pub(super) const MSG_GETTING_PUBLIC_STORE_CONFIG: &str = "Getting public store configuration..."; pub(super) const MSG_CREATING_GCS_BUCKET_SPINNER: &str = "Creating GCS bucket..."; pub(super) const MSG_SAVE_TO_PUBLIC_BUCKET_PROMPT: &str = "Do you want to save to public bucket?"; +pub(super) const MSG_ROUND_SELECT_PROMPT: &str = "Select the round to run"; +pub(super) const MSG_WITNESS_GENERATOR_ROUND_ERR: &str = "Witness generator round not found"; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 83da0e1153f903063fefdbe6c010dd79081b587e Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 12:27:26 +0200 Subject: [PATCH 40/87] Add run_compressor --- prover/prover_fri/README.md | 2 +- .../zk_inception/src/commands/prover/run.rs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index 5f0a26cfdd49..f8626eb7de31 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -202,7 +202,7 @@ There is an option to run compressors with the GPU, which will significantly imp 6. Run the compressor using: ```console - zk f cargo run ---features "gpu" --release --bin zksync_proof_fri_compressor + zk f cargo run --features "gpu" --release --bin zksync_proof_fri_compressor ``` ## Checking the status of the prover diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 975fe3d500f6..8c7fb0423bd1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -8,11 +8,11 @@ use super::{ utils::get_link_to_prover, }; use crate::messages::{ - MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, MSG_RUNNING_PROVER, - MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, - MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, - MSG_WITNESS_GENERATOR_ROUND_ERR, + MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, + MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR, + MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, + MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, MSG_WITNESS_GENERATOR_ROUND_ERR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -99,5 +99,10 @@ fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { logger::info(MSG_RUNNING_COMPRESSOR); - todo!() + let config_path = chain.path_to_general_config(); + let secrets_path = chain.path_to_secrets_config(); + + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin --features zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); + cmd = cmd.with_force_run(); + cmd.run().context(MSG_RUNNING_COMPRESSOR_ERR) } From 2112fd43e48b494fe48eaa2d9858e834a8333202 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 12:27:39 +0200 Subject: [PATCH 41/87] Fix run_compressor --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 8c7fb0423bd1..80ebb0bec4fb 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -102,7 +102,7 @@ fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin --features zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin --features gpu zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_COMPRESSOR_ERR) } From 89e83e3ae6197ba3595a58f6dd036f68b2930a78 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 13:04:39 +0200 Subject: [PATCH 42/87] Remove unused function --- zk_toolbox/crates/common/src/cmd.rs | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/zk_toolbox/crates/common/src/cmd.rs b/zk_toolbox/crates/common/src/cmd.rs index d27dbe303922..4f69a238faab 100644 --- a/zk_toolbox/crates/common/src/cmd.rs +++ b/zk_toolbox/crates/common/src/cmd.rs @@ -1,8 +1,4 @@ -use std::{ - ffi::OsStr, - path::PathBuf, - process::{Command, Output, Stdio}, -}; +use std::{ffi::OsStr, process::Output}; use anyhow::bail; use console::style; @@ -157,15 +153,3 @@ fn get_indented_output( indent(&wrap_text_to_len(&stderr)), ) } - -pub fn run_in_background(command: &str, wd: &PathBuf) -> anyhow::Result { - let cmd = Command::new("sh") - .current_dir(wd) - .arg("-c") - .arg(command) - .stdout(Stdio::inherit()) - .stderr(Stdio::inherit()) - .spawn()?; - - Ok(cmd.id()) -} From b0b49d10c676d6848040fe615ccbc83b53bcb476 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 1 Jul 2024 14:19:23 +0200 Subject: [PATCH 43/87] Fix download_setup_key --- .../zk_inception/src/commands/prover/init.rs | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index b24b470b6390..d0f667443917 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -30,7 +30,7 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( .get_zksync_general_config() .expect(MSG_GENERAL_CONFIG_NOT_FOUND_ERR); - let setup_key_path = get_setup_key_path(&general_config, &ecosystem_config)?; + let setup_key_path = get_setup_key_path(&ecosystem_config)?; let args = args.fill_values_with_prompt(shell, &setup_key_path)?; @@ -82,24 +82,15 @@ fn download_setup_key( .clone(); let url = compressor_config.universal_setup_download_url; - let mut cmd = Cmd::new(cmd!(shell, "wget {url} -P {path}")); + let mut cmd = Cmd::new(cmd!(shell, "wget {url} -O {path}")); cmd.run()?; spinner.finish(); Ok(()) } -fn get_setup_key_path( - general_config: &GeneralConfig, - ecosystem_config: &EcosystemConfig, -) -> anyhow::Result { - let setup_key_path = general_config - .proof_compressor_config - .as_ref() - .expect(MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR) - .universal_setup_path - .clone(); +fn get_setup_key_path(ecosystem_config: &EcosystemConfig) -> anyhow::Result { let link_to_prover = get_link_to_prover(ecosystem_config); - let path = link_to_prover.join(setup_key_path); + let path = link_to_prover.join("keys/setup/setup_2^24.key"); let string = path.to_str().unwrap(); Ok(String::from(string)) From 3be1b70ac2088cd47c9629f5f207fdbfdc2c6efc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 09:11:17 +0200 Subject: [PATCH 44/87] Update genesis protocol version --- etc/env/file_based/genesis.yaml | 2 +- prover/prover_fri/README.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/etc/env/file_based/genesis.yaml b/etc/env/file_based/genesis.yaml index e3513a8b6421..329208bc1e49 100644 --- a/etc/env/file_based/genesis.yaml +++ b/etc/env/file_based/genesis.yaml @@ -1,7 +1,7 @@ genesis_root: 0xabdb766b18a479a5c783a4b80e12686bc8ea3cc2d8a3050491b701d72370ebb5 genesis_rollup_leaf_index: 54 genesis_batch_commitment: 0x2d00e5f8d77afcebf58a6b82ae56ba967566fe7dfbcb6760319fb0d215d18ffd -genesis_protocol_semantic_version: '0.24.0' +genesis_protocol_semantic_version: '0.24.1' # deprecated genesis_protocol_version: 24 default_aa_hash: 0x01000563374c277a2c1e34659a2a1e87371bb6d852ce142022d497bfb50b9e32 diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index f8626eb7de31..5156aeed3bcc 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -176,8 +176,8 @@ There is an option to run compressors with the GPU, which will significantly imp 2. Install and compile `era-bellman-cuda` library ```console - git clone https://github.com/matter-labs/bellman-cuda.git --branch dev bellman-cuda - cmake -Bbellman-cuda/build -Sbellman-cuda/ -DCMAKE_BUILD_TYPE=Release + git clone https://github.com/matter-labs/era-bellman-cuda + cmake -Bbellman-cuda/build -Sera-bellman-cuda/ -DCMAKE_BUILD_TYPE=Release cmake --build bellman-cuda/build/ ``` From d021255a29a116d2fc0d9c017d52ee27cdcdda40 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 09:19:58 +0200 Subject: [PATCH 45/87] Fix setup_key_path --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 7 +++++-- zk_toolbox/crates/zk_inception/src/messages.rs | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index d0f667443917..62cc2dd22b8b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -16,7 +16,7 @@ use crate::{ messages::{ MSG_CHAIN_NOT_FOUND_ERR, MSG_DOWNLOADING_SETUP_KEY_SPINNER, MSG_GENERAL_CONFIG_NOT_FOUND_ERR, MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR, - MSG_PROVER_CONFIG_NOT_FOUND_ERR, MSG_PROVER_INITIALIZED, + MSG_PROVER_CONFIG_NOT_FOUND_ERR, MSG_PROVER_INITIALIZED, MSG_SETUP_KEY_PATH_ERROR, }, }; @@ -81,8 +81,11 @@ fn download_setup_key( .expect(MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR) .clone(); let url = compressor_config.universal_setup_download_url; + let parent = std::path::Path::new(path) + .parent() + .expect(MSG_SETUP_KEY_PATH_ERROR); - let mut cmd = Cmd::new(cmd!(shell, "wget {url} -O {path}")); + let mut cmd = Cmd::new(cmd!(shell, "wget {url} -P {parent}")); cmd.run()?; spinner.finish(); Ok(()) diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index a25f3fc9baa5..4c4c59c7b8a9 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -259,6 +259,7 @@ pub(super) const MSG_CREATING_GCS_BUCKET_SPINNER: &str = "Creating GCS bucket... pub(super) const MSG_SAVE_TO_PUBLIC_BUCKET_PROMPT: &str = "Do you want to save to public bucket?"; pub(super) const MSG_ROUND_SELECT_PROMPT: &str = "Select the round to run"; pub(super) const MSG_WITNESS_GENERATOR_ROUND_ERR: &str = "Witness generator round not found"; +pub(super) const MSG_SETUP_KEY_PATH_ERROR: &str = "Failed to get setup key path"; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 962c5401301764193bd28728ad2b9ff650185322 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 11:40:07 +0200 Subject: [PATCH 46/87] Add static KEYSTORE --- .../src/commitment_utils.rs | 18 ++++++++++++++---- .../src/keystore.rs | 9 +++++++++ prover/witness_generator/src/main.rs | 11 ++++++----- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs index 935d06460181..760fe8257608 100644 --- a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs +++ b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs @@ -1,4 +1,4 @@ -use std::str::FromStr; +use std::{str::FromStr, sync::Mutex}; use anyhow::Context as _; use hex::ToHex; @@ -22,9 +22,14 @@ use crate::{ VkCommitments, }; +static KEYSTORE: Lazy>> = Lazy::new(|| Mutex::new(None)); + lazy_static! { // TODO: do not initialize a static const with data read in runtime. - static ref COMMITMENTS: Lazy = Lazy::new(|| { circuit_commitments(&Keystore::default()).unwrap() }); + static ref COMMITMENTS: Lazy = Lazy::new(|| { + let keystore = KEYSTORE.lock().unwrap().clone().unwrap_or_else(Keystore::default); + circuit_commitments(&keystore).unwrap() + }); } fn circuit_commitments(keystore: &Keystore) -> anyhow::Result { @@ -97,14 +102,19 @@ pub fn generate_commitments(keystore: &Keystore) -> anyhow::Result L1VerifierConfig { +pub fn get_cached_commitments(setup_data_generator: Option) -> L1VerifierConfig { + if let Some(setup_data_generator) = setup_data_generator { + let keystore = Keystore::new_with_setup_data_path(setup_data_generator); + let mut keystore_lock = KEYSTORE.lock().unwrap(); + *keystore_lock = Some(keystore); + } tracing::info!("Using cached commitments {:?}", **COMMITMENTS); **COMMITMENTS } #[test] fn test_get_cached_commitments() { - let commitments = get_cached_commitments(); + let commitments = get_cached_commitments(None); assert_eq!( H256::zero(), commitments.params.recursion_circuits_set_vks_hash diff --git a/prover/vk_setup_data_generator_server_fri/src/keystore.rs b/prover/vk_setup_data_generator_server_fri/src/keystore.rs index 25aedeb089ff..70aaff9fc4a4 100644 --- a/prover/vk_setup_data_generator_server_fri/src/keystore.rs +++ b/prover/vk_setup_data_generator_server_fri/src/keystore.rs @@ -36,6 +36,7 @@ pub enum ProverServiceDataType { /// There are 2 types: /// - small verification, finalization keys (used only during verification) /// - large setup keys, used during proving. +#[derive(Clone)] pub struct Keystore { /// Directory to store all the small keys. basedir: PathBuf, @@ -80,6 +81,7 @@ impl Keystore { setup_data_path: Some(setup_data_path), } } + pub fn new_with_optional_setup_path(basedir: PathBuf, setup_data_path: Option) -> Self { Keystore { basedir, @@ -87,6 +89,13 @@ impl Keystore { } } + pub fn new_with_setup_data_path(setup_data_path: String) -> Self { + Keystore { + basedir: get_base_path(), + setup_data_path: Some(setup_data_path), + } + } + pub fn get_base_path(&self) -> &PathBuf { &self.basedir } diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 8208c62c6277..661965b75061 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -111,12 +111,12 @@ async fn main() -> anyhow::Result<()> { let started_at = Instant::now(); let use_push_gateway = opt.batch_size.is_some(); + let prover_config = general_config.prover_config.context("prover config")?; let object_store_config = ProverObjectStoreConfig( - general_config - .prover_config - .context("prover config")? + prover_config .prover_object_store - .context("object store")?, + .context("object store")? + .clone(), ); let store_factory = ObjectStoreFactory::new(object_store_config.0); let config = general_config @@ -202,7 +202,8 @@ async fn main() -> anyhow::Result<()> { let witness_generator_task = match round { AggregationRound::BasicCircuits => { - let vk_commitments = get_cached_commitments(); + let setup_data_path = prover_config.setup_data_path.clone(); + let vk_commitments = get_cached_commitments(Some(setup_data_path)); assert_eq!( vk_commitments, vk_commitments_in_db, From 774f932db8d1caf164aaeef9b6280af7d75111e8 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 12:04:17 +0200 Subject: [PATCH 47/87] Update variable name --- .../src/commitment_utils.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs index 760fe8257608..8a55930124e2 100644 --- a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs +++ b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs @@ -102,9 +102,9 @@ pub fn generate_commitments(keystore: &Keystore) -> anyhow::Result) -> L1VerifierConfig { - if let Some(setup_data_generator) = setup_data_generator { - let keystore = Keystore::new_with_setup_data_path(setup_data_generator); +pub fn get_cached_commitments(setup_data_path: Option) -> L1VerifierConfig { + if let Some(setup_data_path) = setup_data_path { + let keystore = Keystore::new_with_setup_data_path(setup_data_path); let mut keystore_lock = KEYSTORE.lock().unwrap(); *keystore_lock = Some(keystore); } From df5907fb50e7b66c0861e7bdd6606fcab3befdd1 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 12:21:32 +0200 Subject: [PATCH 48/87] lint --- .../vk_setup_data_generator_server_fri/src/commitment_utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs index 8a55930124e2..58fd36ab4a59 100644 --- a/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs +++ b/prover/vk_setup_data_generator_server_fri/src/commitment_utils.rs @@ -27,7 +27,7 @@ static KEYSTORE: Lazy>> = Lazy::new(|| Mutex::new(None)); lazy_static! { // TODO: do not initialize a static const with data read in runtime. static ref COMMITMENTS: Lazy = Lazy::new(|| { - let keystore = KEYSTORE.lock().unwrap().clone().unwrap_or_else(Keystore::default); + let keystore = KEYSTORE.lock().unwrap().clone().unwrap_or_default(); circuit_commitments(&keystore).unwrap() }); } From 03b6ab85f0bcdc4e89eb47a94f70f5e274afb7d3 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 14:22:08 +0200 Subject: [PATCH 49/87] Update vk hashes --- etc/env/file_based/genesis.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/genesis.yaml b/etc/env/file_based/genesis.yaml index 329208bc1e49..4f084648c7ca 100644 --- a/etc/env/file_based/genesis.yaml +++ b/etc/env/file_based/genesis.yaml @@ -10,9 +10,9 @@ l1_chain_id: 9 l2_chain_id: 270 fee_account: '0x0000000000000000000000000000000000000001' prover: - recursion_scheduler_level_vk_hash: 0x712bb009b5d5dc81c79f827ca0abff87b43506a8efed6028a818911d4b1b521f + recursion_scheduler_level_vk_hash: 0x14f97b81e54b35fe673d8708cc1a19e1ea5b5e348e12d31e39824ed4f42bbca2 recursion_node_level_vk_hash: 0xf520cd5b37e74e19fdb369c8d676a04dce8a19457497ac6686d2bb95d94109c8 - recursion_leaf_level_vk_hash: 0xffb19d007c67b9000b40b372e7a7a55a47d11c92588515598d6cad4052c75ebb + recursion_leaf_level_vk_hash: 0xf9664f4324c1400fa5c3822d667f30e873f53f1b8033180cd15fe41c1e2355c6 recursion_circuits_set_vks_hash: '0x0000000000000000000000000000000000000000000000000000000000000000' dummy_verifier: true l1_batch_commit_data_generator_mode: Rollup From f359b4f47e44f5b7942705605dc008344d759c75 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 14:59:15 +0200 Subject: [PATCH 50/87] Fix public object store --- prover/witness_generator/src/main.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 661965b75061..c4d86b3fb30f 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -214,8 +214,9 @@ async fn main() -> anyhow::Result<()> { false => None, true => Some( ObjectStoreFactory::new( - ObjectStoreConfig::from_env() - .context("ObjectStoreConfig::from_env()")?, + prover_config + .public_object_store + .expect("public_object_store")?, ) .create_store() .await?, From d752fe4e3ea7ebaf28940cf6c4161ea9d4c4020f Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 15:05:50 +0200 Subject: [PATCH 51/87] Fix object store --- prover/witness_generator/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index c4d86b3fb30f..d5e78dce859a 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -6,8 +6,7 @@ use anyhow::{anyhow, Context as _}; use futures::{channel::mpsc, executor::block_on, SinkExt, StreamExt}; use structopt::StructOpt; use tokio::sync::watch; -use zksync_config::ObjectStoreConfig; -use zksync_env_config::{object_store::ProverObjectStoreConfig, FromEnv}; +use zksync_env_config::object_store::ProverObjectStoreConfig; use zksync_object_store::ObjectStoreFactory; use zksync_prover_config::{load_database_secrets, load_general_config}; use zksync_prover_dal::{ConnectionPool, Prover, ProverDal}; @@ -216,7 +215,8 @@ async fn main() -> anyhow::Result<()> { ObjectStoreFactory::new( prover_config .public_object_store - .expect("public_object_store")?, + .clone() + .expect("public_object_store"), ) .create_store() .await?, From 06d37c044d50f2135d5ee1c774e9ee848b8023ce Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 15:24:57 +0200 Subject: [PATCH 52/87] Update prometheus listener port --- etc/env/file_based/general.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index f2733d5d1ee4..e703964dbcb0 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -19,7 +19,7 @@ db: api: prometheus: - listener_port: 3312 + listener_port: 3316 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 healthcheck: @@ -313,7 +313,7 @@ house_keeper: fri_gpu_prover_archiver_archive_after_secs: 172800 prometheus: - listener_port: 3312 + listener_port: 3316 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 From 15b4c929a8516561fc6e806ea36ba052f3a118a5 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 15:35:32 +0200 Subject: [PATCH 53/87] Update prometheus ports --- etc/env/file_based/general.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index e703964dbcb0..57c066ae3f8c 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -19,7 +19,7 @@ db: api: prometheus: - listener_port: 3316 + listener_port: 3116 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 healthcheck: @@ -313,7 +313,7 @@ house_keeper: fri_gpu_prover_archiver_archive_after_secs: 172800 prometheus: - listener_port: 3316 + listener_port: 3116 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 From 213ebf8e633f382992742e8e3b24cd4ff1078430 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 2 Jul 2024 16:11:58 +0200 Subject: [PATCH 54/87] Fix witness vector generator file based config --- etc/env/file_based/general.yaml | 2 +- prover/witness_vector_generator/src/generator.rs | 12 +++++++++++- prover/witness_vector_generator/src/main.rs | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 57c066ae3f8c..23e741670043 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -184,7 +184,7 @@ witness_generator: witness_vector_generator: prover_instance_wait_timeout_in_secs: 200 prover_instance_poll_time_in_milli_secs: 250 - prometheus_listener_port: 3314 + prometheus_listener_port: 3420 prometheus_pushgateway_url: http://127.0.0.1:9091 prometheus_push_interval_ms: 100 specialized_group_id: 100 diff --git a/prover/witness_vector_generator/src/generator.rs b/prover/witness_vector_generator/src/generator.rs index b7b9dcd9f763..ba6a7ad30e21 100644 --- a/prover/witness_vector_generator/src/generator.rs +++ b/prover/witness_vector_generator/src/generator.rs @@ -34,6 +34,7 @@ pub struct WitnessVectorGenerator { config: FriWitnessVectorGeneratorConfig, protocol_version: ProtocolSemanticVersion, max_attempts: u32, + setup_data_path: Option, } impl WitnessVectorGenerator { @@ -45,6 +46,7 @@ impl WitnessVectorGenerator { config: FriWitnessVectorGeneratorConfig, protocol_version: ProtocolSemanticVersion, max_attempts: u32, + setup_data_path: Option, ) -> Self { Self { object_store, @@ -54,6 +56,7 @@ impl WitnessVectorGenerator { config, protocol_version, max_attempts, + setup_data_path, } } @@ -116,10 +119,17 @@ impl JobProcessor for WitnessVectorGenerator { job: ProverJob, _started_at: Instant, ) -> JoinHandle> { + let setup_data_path = self.setup_data_path.clone(); + tokio::task::spawn_blocking(move || { let block_number = job.block_number; let _span = tracing::info_span!("witness_vector_generator", %block_number).entered(); - Self::generate_witness_vector(job, &Keystore::default()) + let keystore = if let Some(setup_data_path) = setup_data_path { + Keystore::new_with_setup_data_path(setup_data_path) + } else { + Keystore::default() + }; + Self::generate_witness_vector(job, &keystore) }) } diff --git a/prover/witness_vector_generator/src/main.rs b/prover/witness_vector_generator/src/main.rs index 1649c8e82acd..a7ade8b36b8d 100644 --- a/prover/witness_vector_generator/src/main.rs +++ b/prover/witness_vector_generator/src/main.rs @@ -108,6 +108,7 @@ async fn main() -> anyhow::Result<()> { config, protocol_version, fri_prover_config.max_attempts, + Some(fri_prover_config.setup_data_path.clone()), ); let (stop_sender, stop_receiver) = watch::channel(false); From 245047b85191801f15d23bba56e06dee96e427bf Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 3 Jul 2024 15:30:37 +0200 Subject: [PATCH 55/87] Fix run_compressor --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 80ebb0bec4fb..586e1a45805c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -102,7 +102,7 @@ fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let mut cmd = Cmd::new(cmd!(shell, "cargo run --release --bin --features gpu zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --features gpu --release --bin zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_COMPRESSOR_ERR) } From 0316019d70d5f436cf779c17025ae89f865943fc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Wed, 3 Jul 2024 16:17:55 +0200 Subject: [PATCH 56/87] Add init_bellman_cuda --- zk_toolbox/crates/common/src/prerequisites.rs | 18 +++++++-- .../zk_inception/src/commands/prover/init.rs | 38 +++++++++++++++++-- .../crates/zk_inception/src/messages.rs | 2 + 3 files changed, 50 insertions(+), 8 deletions(-) diff --git a/zk_toolbox/crates/common/src/prerequisites.rs b/zk_toolbox/crates/common/src/prerequisites.rs index 717635a1a18f..8901add90233 100644 --- a/zk_toolbox/crates/common/src/prerequisites.rs +++ b/zk_toolbox/crates/common/src/prerequisites.rs @@ -30,10 +30,20 @@ const DOCKER_COMPOSE_PREREQUISITE: Prerequisite = Prerequisite { download_link: "https://docs.docker.com/compose/install/", }; -const PROVER_PREREQUISITES: [Prerequisite; 1] = [Prerequisite { - name: "gcloud", - download_link: "https://cloud.google.com/sdk/docs/install", -}]; +const PROVER_PREREQUISITES: [Prerequisite; 3] = [ + Prerequisite { + name: "gcloud", + download_link: "https://cloud.google.com/sdk/docs/install", + }, + Prerequisite { + name: "wget", + download_link: "https://www.gnu.org/software/wget/", + }, + Prerequisite { + name: "cmake", + download_link: "https://cmake.org/download/", + }, +]; struct Prerequisite { name: &'static str, diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 297933114c7a..38e95f8814de 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -1,4 +1,11 @@ -use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; +use std::path::PathBuf; + +use common::{ + check_prover_prequisites, + cmd::Cmd, + logger, + spinner::{self, Spinner}, +}; use config::EcosystemConfig; use xshell::{cmd, Shell}; use zksync_config::{ @@ -14,7 +21,8 @@ use super::{ use crate::{ consts::PROVER_STORE_MAX_RETRIES, messages::{ - MSG_CHAIN_NOT_FOUND_ERR, MSG_DOWNLOADING_SETUP_KEY_SPINNER, + MSG_BUILDING_BELLMAN_CUDA_SPINNER, MSG_CHAIN_NOT_FOUND_ERR, + MSG_CLONING_BELLMAN_CUDA_SPINNER, MSG_DOWNLOADING_SETUP_KEY_SPINNER, MSG_GENERAL_CONFIG_NOT_FOUND_ERR, MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR, MSG_PROVER_CONFIG_NOT_FOUND_ERR, MSG_PROVER_INITIALIZED, MSG_SETUP_KEY_PATH_ERROR, }, @@ -67,6 +75,8 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( chain_config.save_zksync_general_config(&general_config)?; + init_bellman_cuda(shell)?; + logger::outro(MSG_PROVER_INITIALIZED); Ok(()) } @@ -87,8 +97,8 @@ fn download_setup_key( .parent() .expect(MSG_SETUP_KEY_PATH_ERROR); - let mut cmd = Cmd::new(cmd!(shell, "wget {url} -P {parent}")); - cmd.run()?; + Cmd::new(cmd!(shell, "wget {url} -P {parent}")).run()?; + spinner.finish(); Ok(()) } @@ -129,3 +139,23 @@ fn get_object_store_config( Ok(object_store) } + +fn init_bellman_cuda(shell: &Shell) -> anyhow::Result<()> { + let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); + Cmd::new(cmd!( + shell, + "git clone https://github.com/matter-labs/era-bellman-cuda" + )) + .run()?; + spinner.finish(); + + let spinner = Spinner::new(MSG_BUILDING_BELLMAN_CUDA_SPINNER); + Cmd::new(cmd!( + shell, + "cmake -Bera-bellman-cuda/build -Sera-bellman-cuda/ -DCMAKE_BUILD_TYPE=Release" + )) + .run()?; + Cmd::new(cmd!(shell, "cmake --build era-bellman-cuda/build")).run()?; + spinner.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 4c4c59c7b8a9..0900867d3171 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -260,6 +260,8 @@ pub(super) const MSG_SAVE_TO_PUBLIC_BUCKET_PROMPT: &str = "Do you want to save t pub(super) const MSG_ROUND_SELECT_PROMPT: &str = "Select the round to run"; pub(super) const MSG_WITNESS_GENERATOR_ROUND_ERR: &str = "Witness generator round not found"; pub(super) const MSG_SETUP_KEY_PATH_ERROR: &str = "Failed to get setup key path"; +pub(super) const MSG_CLONING_BELLMAN_CUDA_SPINNER: &str = "Cloning bellman-cuda..."; +pub(super) const MSG_BUILDING_BELLMAN_CUDA_SPINNER: &str = "Building bellman-cuda..."; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 7fc56e18a7dd3367348ef764536fbb7528246192 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 11:57:44 +0200 Subject: [PATCH 57/87] Add BELLMAN_CUDA_DIR variable --- zk_toolbox/Cargo.lock | 1 + zk_toolbox/crates/zk_inception/Cargo.toml | 1 + .../zk_inception/src/commands/prover/init.rs | 15 +++++------- .../zk_inception/src/commands/prover/run.rs | 23 ++++++++++++++----- zk_toolbox/crates/zk_inception/src/consts.rs | 1 + .../crates/zk_inception/src/messages.rs | 1 + 6 files changed, 27 insertions(+), 15 deletions(-) diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock index 0dbb9cf12463..9dca0fd485b9 100644 --- a/zk_toolbox/Cargo.lock +++ b/zk_toolbox/Cargo.lock @@ -6313,6 +6313,7 @@ dependencies = [ "ethers", "human-panic", "lazy_static", + "path-absolutize", "serde", "serde_json", "serde_yaml", diff --git a/zk_toolbox/crates/zk_inception/Cargo.toml b/zk_toolbox/crates/zk_inception/Cargo.toml index 3a8b57e162f6..3a4ebf0f622c 100644 --- a/zk_toolbox/crates/zk_inception/Cargo.toml +++ b/zk_toolbox/crates/zk_inception/Cargo.toml @@ -16,6 +16,7 @@ clap.workspace = true cliclack.workspace = true config.workspace = true console.workspace = true +path-absolutize.workspace = true human-panic.workspace = true lazy_static.workspace = true serde_yaml.workspace = true diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 38e95f8814de..9b681103b21a 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -1,11 +1,4 @@ -use std::path::PathBuf; - -use common::{ - check_prover_prequisites, - cmd::Cmd, - logger, - spinner::{self, Spinner}, -}; +use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; use zksync_config::{ @@ -19,7 +12,7 @@ use super::{ utils::get_link_to_prover, }; use crate::{ - consts::PROVER_STORE_MAX_RETRIES, + consts::{BELLMAN_CUDA_DIR, PROVER_STORE_MAX_RETRIES}, messages::{ MSG_BUILDING_BELLMAN_CUDA_SPINNER, MSG_CHAIN_NOT_FOUND_ERR, MSG_CLONING_BELLMAN_CUDA_SPINNER, MSG_DOWNLOADING_SETUP_KEY_SPINNER, @@ -141,6 +134,10 @@ fn get_object_store_config( } fn init_bellman_cuda(shell: &Shell) -> anyhow::Result<()> { + if shell.path_exists(BELLMAN_CUDA_DIR) { + return Ok(()); + } + let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); Cmd::new(cmd!( shell, diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 586e1a45805c..c56cd60d790b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,18 +1,24 @@ use anyhow::Context; use common::{cmd::Cmd, logger}; use config::{ChainConfig, EcosystemConfig}; +use path_absolutize::Absolutize; +use std::path::PathBuf; use xshell::{cmd, Shell}; use super::{ args::run::{ProverComponent, ProverRunArgs, WitnessGeneratorArgs, WitnessGeneratorRound}, utils::get_link_to_prover, }; -use crate::messages::{ - MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, - MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR, - MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, - MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, MSG_WITNESS_GENERATOR_ROUND_ERR, +use crate::{ + consts::BELLMAN_CUDA_DIR, + messages::{ + MSG_BELLMAN_CUDA_DIR_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, + MSG_RUNNING_COMPRESSOR, MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, + MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, + MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, + MSG_WITNESS_GENERATOR_ROUND_ERR, + }, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -102,6 +108,11 @@ fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); + let bellman_cuda_dir = PathBuf::from(BELLMAN_CUDA_DIR); + let bellman_cuda_dir = bellman_cuda_dir.absolutize()?; + let bellman_cuda_str = bellman_cuda_dir.to_str().expect(MSG_BELLMAN_CUDA_DIR_ERR); + shell.set_var("BELLMAN_CUDA_DIR", bellman_cuda_str); + let mut cmd = Cmd::new(cmd!(shell, "cargo run --features gpu --release --bin zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); cmd.run().context(MSG_RUNNING_COMPRESSOR_ERR) diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 1693ff1d2f41..e0258fb46402 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -6,3 +6,4 @@ pub const PROVER_MIGRATIONS: &str = "prover/prover_dal/migrations"; pub const PROVER_STORE_MAX_RETRIES: u16 = 10; pub const DEFAULT_CREDENTIALS_FILE: &str = "~/.config/gcloud/application_default_credentials.json"; pub const DEFAULT_PROOF_STORE_DIR: &str = "artifacts"; +pub const BELLMAN_CUDA_DIR: &str = "era-bellman-cuda"; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 0900867d3171..4eb0235c2298 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -262,6 +262,7 @@ pub(super) const MSG_WITNESS_GENERATOR_ROUND_ERR: &str = "Witness generator roun pub(super) const MSG_SETUP_KEY_PATH_ERROR: &str = "Failed to get setup key path"; pub(super) const MSG_CLONING_BELLMAN_CUDA_SPINNER: &str = "Cloning bellman-cuda..."; pub(super) const MSG_BUILDING_BELLMAN_CUDA_SPINNER: &str = "Building bellman-cuda..."; +pub(super) const MSG_BELLMAN_CUDA_DIR_ERR: &str = "Failed to get bellman-cuda directory"; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 6a3986a558f65994228e82e0e64e592f50593a57 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 11:58:06 +0200 Subject: [PATCH 58/87] fmt --- zk_toolbox/crates/zk_inception/src/commands/prover/run.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index c56cd60d790b..c5995a7a5826 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,8 +1,9 @@ +use std::path::PathBuf; + use anyhow::Context; use common::{cmd::Cmd, logger}; use config::{ChainConfig, EcosystemConfig}; use path_absolutize::Absolutize; -use std::path::PathBuf; use xshell::{cmd, Shell}; use super::{ From 9bf2d5a1c0a113027228466c7cc39e6ae6e7041d Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:14:12 +0200 Subject: [PATCH 59/87] Update readme --- prover/prover_fri/README.md | 2 +- zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prover/prover_fri/README.md b/prover/prover_fri/README.md index 5156aeed3bcc..f68de6657cea 100644 --- a/prover/prover_fri/README.md +++ b/prover/prover_fri/README.md @@ -177,7 +177,7 @@ There is an option to run compressors with the GPU, which will significantly imp ```console git clone https://github.com/matter-labs/era-bellman-cuda - cmake -Bbellman-cuda/build -Sera-bellman-cuda/ -DCMAKE_BUILD_TYPE=Release + cmake -Bera-bellman-cuda/build -Sera-bellman-cuda/ -DCMAKE_BUILD_TYPE=Release cmake --build bellman-cuda/build/ ``` diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs index 28d5f1bec038..678c548cea64 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/run.rs @@ -54,7 +54,7 @@ pub enum WitnessGeneratorRound { impl ProverRunArgs { pub fn fill_values_with_prompt(&self) -> anyhow::Result { - let component = self.component.clone().unwrap_or_else(|| { + let component = self.component.unwrap_or_else(|| { PromptSelect::new(MSG_RUN_COMPONENT_PROMPT, ProverComponent::iter()).ask() }); From e7f274d640672c0038d5c5d5ae0dfdbb597b2f7e Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:18:13 +0200 Subject: [PATCH 60/87] Add nvidia prerequisite --- zk_toolbox/crates/common/src/prerequisites.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/common/src/prerequisites.rs b/zk_toolbox/crates/common/src/prerequisites.rs index 8901add90233..8b101e86dc0c 100644 --- a/zk_toolbox/crates/common/src/prerequisites.rs +++ b/zk_toolbox/crates/common/src/prerequisites.rs @@ -30,7 +30,7 @@ const DOCKER_COMPOSE_PREREQUISITE: Prerequisite = Prerequisite { download_link: "https://docs.docker.com/compose/install/", }; -const PROVER_PREREQUISITES: [Prerequisite; 3] = [ +const PROVER_PREREQUISITES: [Prerequisite; 4] = [ Prerequisite { name: "gcloud", download_link: "https://cloud.google.com/sdk/docs/install", @@ -43,6 +43,10 @@ const PROVER_PREREQUISITES: [Prerequisite; 3] = [ name: "cmake", download_link: "https://cmake.org/download/", }, + Prerequisite { + name: "nvcc", + download_link: "https://developer.nvidia.com/cuda-downloads", + }, ]; struct Prerequisite { From 0cc9ca8712c3367602575b2225b6dc91ff7ca9bc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:21:34 +0200 Subject: [PATCH 61/87] Update strum version --- zk_toolbox/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/Cargo.toml b/zk_toolbox/Cargo.toml index 1a6a69b207d0..f262fdbe617c 100644 --- a/zk_toolbox/Cargo.toml +++ b/zk_toolbox/Cargo.toml @@ -45,7 +45,7 @@ serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde_yaml = "0.9" sqlx = { version = "0.7.4", features = ["runtime-tokio", "migrate", "postgres"] } -strum = { workspace = true, features = ["derive"] } +strum = { version = "0.26.2", features = ["derive"] } strum_macros = "0.26.2" thiserror = "1.0.57" tokio = { version = "1.37", features = ["full"] } From ce41b12cc2d355886462338e40a0101606ea746c Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:23:14 +0200 Subject: [PATCH 62/87] Update get_default_setup_key_path --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 9b681103b21a..57290b020fd5 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -31,7 +31,7 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( .get_zksync_general_config() .expect(MSG_GENERAL_CONFIG_NOT_FOUND_ERR); - let setup_key_path = get_setup_key_path(&ecosystem_config)?; + let setup_key_path = get_default_setup_key_path(&ecosystem_config)?; let args = args.fill_values_with_prompt(shell, &setup_key_path)?; @@ -96,7 +96,7 @@ fn download_setup_key( Ok(()) } -fn get_setup_key_path(ecosystem_config: &EcosystemConfig) -> anyhow::Result { +fn get_default_setup_key_path(ecosystem_config: &EcosystemConfig) -> anyhow::Result { let link_to_prover = get_link_to_prover(ecosystem_config); let path = link_to_prover.join("keys/setup/setup_2^24.key"); let string = path.to_str().unwrap(); From 3cbb8d02f334c1899816a20094091b4f70445f02 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:24:43 +0200 Subject: [PATCH 63/87] Update download_setup_key --- .../crates/zk_inception/src/commands/prover/init.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 57290b020fd5..470fcff34333 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -86,11 +86,11 @@ fn download_setup_key( .expect(MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR) .clone(); let url = compressor_config.universal_setup_download_url; - let parent = std::path::Path::new(path) - .parent() - .expect(MSG_SETUP_KEY_PATH_ERROR); + let path = std::path::Path::new(path); + let parent = path.parent().expect(MSG_SETUP_KEY_PATH_ERROR); + let file_name = path.file_name().expect(MSG_SETUP_KEY_PATH_ERROR); - Cmd::new(cmd!(shell, "wget {url} -P {parent}")).run()?; + Cmd::new(cmd!(shell, "wget {url} -P {parent} -O {file_name}")).run()?; spinner.finish(); Ok(()) From 225479bb928d31d3a501e963be1567d74c57ba58 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:30:27 +0200 Subject: [PATCH 64/87] Fix download_setup_key --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 470fcff34333..2b816a79162a 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -90,7 +90,8 @@ fn download_setup_key( let parent = path.parent().expect(MSG_SETUP_KEY_PATH_ERROR); let file_name = path.file_name().expect(MSG_SETUP_KEY_PATH_ERROR); - Cmd::new(cmd!(shell, "wget {url} -P {parent} -O {file_name}")).run()?; + Cmd::new(cmd!(shell, "wget {url} -P {parent}")).run()?; + Cmd::new(cmd!(shell, "mv {parent}/setup_2^24.key {path}")).run()?; spinner.finish(); Ok(()) From a457c37490a384c7bd03e3f5017902ebc24a3d57 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:33:13 +0200 Subject: [PATCH 65/87] Remove unused variable --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 2b816a79162a..4797f81d2fb8 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -88,7 +88,6 @@ fn download_setup_key( let url = compressor_config.universal_setup_download_url; let path = std::path::Path::new(path); let parent = path.parent().expect(MSG_SETUP_KEY_PATH_ERROR); - let file_name = path.file_name().expect(MSG_SETUP_KEY_PATH_ERROR); Cmd::new(cmd!(shell, "wget {url} -P {parent}")).run()?; Cmd::new(cmd!(shell, "mv {parent}/setup_2^24.key {path}")).run()?; From 1c058996ab7102d56a18e7ca09589ab1d7449a78 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 12:46:35 +0200 Subject: [PATCH 66/87] lint --- prover/witness_vector_generator/src/generator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/prover/witness_vector_generator/src/generator.rs b/prover/witness_vector_generator/src/generator.rs index ba6a7ad30e21..d2b13beccd61 100644 --- a/prover/witness_vector_generator/src/generator.rs +++ b/prover/witness_vector_generator/src/generator.rs @@ -38,6 +38,7 @@ pub struct WitnessVectorGenerator { } impl WitnessVectorGenerator { + #[allow(clippy::too_many_arguments)] pub fn new( object_store: Arc, prover_connection_pool: ConnectionPool, From b908055aaf57d70f18f2dbbda8ef38a5804601ce Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 15:59:05 +0200 Subject: [PATCH 67/87] Add prometheus_listener_port to FriWitnessGeneratorConfig --- core/lib/config/src/configs/fri_witness_generator.rs | 2 ++ core/lib/config/src/testonly.rs | 1 + core/lib/env_config/src/fri_witness_generator.rs | 2 ++ core/lib/protobuf_config/src/proto/config/prover.proto | 1 + core/lib/protobuf_config/src/prover.rs | 6 ++++++ 5 files changed, 12 insertions(+) diff --git a/core/lib/config/src/configs/fri_witness_generator.rs b/core/lib/config/src/configs/fri_witness_generator.rs index c69d04367cf8..281159271dd0 100644 --- a/core/lib/config/src/configs/fri_witness_generator.rs +++ b/core/lib/config/src/configs/fri_witness_generator.rs @@ -23,6 +23,8 @@ pub struct FriWitnessGeneratorConfig { // whether to write to public GCS bucket for https://github.com/matter-labs/era-boojum-validator-cli pub shall_save_to_public_bucket: bool, + + pub prometheus_listener_port: Option, } #[derive(Debug)] diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 8db71e2c8e7f..ede6f6de943c 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -559,6 +559,7 @@ impl Distribution for EncodeDist { max_attempts: self.sample(rng), last_l1_batch_to_process: self.sample(rng), shall_save_to_public_bucket: self.sample(rng), + prometheus_listener_port: self.sample(rng), } } } diff --git a/core/lib/env_config/src/fri_witness_generator.rs b/core/lib/env_config/src/fri_witness_generator.rs index 9780e6aec682..5853a0178308 100644 --- a/core/lib/env_config/src/fri_witness_generator.rs +++ b/core/lib/env_config/src/fri_witness_generator.rs @@ -26,6 +26,7 @@ mod tests { max_attempts: 4, last_l1_batch_to_process: None, shall_save_to_public_bucket: true, + prometheus_listener_port: Some(3333u16), } } @@ -41,6 +42,7 @@ mod tests { FRI_WITNESS_SCHEDULER_GENERATION_TIMEOUT_IN_SECS=900 FRI_WITNESS_MAX_ATTEMPTS=4 FRI_WITNESS_SHALL_SAVE_TO_PUBLIC_BUCKET=true + FRI_WITNESS_PROMETHEUS_LISTENER_PORT=3333 "#; lock.set_env(config); diff --git a/core/lib/protobuf_config/src/proto/config/prover.proto b/core/lib/protobuf_config/src/proto/config/prover.proto index 1eaf8637522a..c50ebdde4eef 100644 --- a/core/lib/protobuf_config/src/proto/config/prover.proto +++ b/core/lib/protobuf_config/src/proto/config/prover.proto @@ -81,6 +81,7 @@ message WitnessGenerator { optional uint32 node_generation_timeout_in_secs = 10; // optional; optional uint32 scheduler_generation_timeout_in_secs = 11; // optional; optional uint32 recursion_tip_timeout_in_secs = 12; // optional; + optional uint32 prometheus_listener_port = 13; // optional; reserved 3, 4, 6; reserved "dump_arguments_for_blocks", "force_process_block", "blocks_proving_percentage"; } diff --git a/core/lib/protobuf_config/src/prover.rs b/core/lib/protobuf_config/src/prover.rs index 9a41e433433c..50782ab8e968 100644 --- a/core/lib/protobuf_config/src/prover.rs +++ b/core/lib/protobuf_config/src/prover.rs @@ -193,6 +193,11 @@ impl ProtoRepr for proto::WitnessGenerator { .map(|x| x.try_into()) .transpose() .context("scheduler_generation_timeout_in_secs")?, + prometheus_listener_port: self + .prometheus_listener_port + .map(|x| x.try_into()) + .transpose() + .context("prometheus_listener_port")?, }) } @@ -213,6 +218,7 @@ impl ProtoRepr for proto::WitnessGenerator { scheduler_generation_timeout_in_secs: this .scheduler_generation_timeout_in_secs .map(|x| x.into()), + prometheus_listener_port: this.prometheus_listener_port.map(|x| x.into()), } } } From 8357145225c103df930e7d0c23a110e16feea065 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:05:14 +0200 Subject: [PATCH 68/87] Update prometheus_listener_port --- prover/witness_generator/src/main.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 661965b75061..6d260ba9bcc9 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -122,9 +122,12 @@ async fn main() -> anyhow::Result<()> { let config = general_config .witness_generator .context("witness generator config")?; - let prometheus_config = general_config + let mut prometheus_config = general_config .prometheus_config .context("prometheus config")?; + if let Some(prometheus_listener_port) = config.prometheus_listener_port { + prometheus_config.listener_port = prometheus_listener_port; + } let postgres_config = general_config.postgres_config.context("postgres config")?; let connection_pool = ConnectionPool::::builder( database_secrets.master_url()?, From c136429c278518a18018c2e1fca6005793be18e0 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:08:39 +0200 Subject: [PATCH 69/87] Add prometheus_listener_port to general config --- etc/env/file_based/general.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 4a258a7cd99d..adece364f7a8 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -181,6 +181,7 @@ witness_generator: generation_timeout_in_secs: 900 max_attempts: 10 shall_save_to_public_bucket: true + prometheus_listener_port: 3116 witness_vector_generator: prover_instance_wait_timeout_in_secs: 200 prover_instance_poll_time_in_milli_secs: 250 From 86adc0df06860b70dbc0978114aa8c5a3df04cf0 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:35:33 +0200 Subject: [PATCH 70/87] Add prometheus_listener_port variable --- prover/witness_generator/src/main.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 6d260ba9bcc9..48fd5445c1af 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -125,9 +125,14 @@ async fn main() -> anyhow::Result<()> { let mut prometheus_config = general_config .prometheus_config .context("prometheus config")?; - if let Some(prometheus_listener_port) = config.prometheus_listener_port { - prometheus_config.listener_port = prometheus_listener_port; - } + + // If the prometheus listener port is not set in the witness generator config, use the one from the prometheus config. + let prometheus_listener_port = if let Some(port) = config.prometheus_listener_port { + port; + } else { + prometheus_config.listener_port + }; + let postgres_config = general_config.postgres_config.context("postgres config")?; let connection_pool = ConnectionPool::::builder( database_secrets.master_url()?, @@ -199,7 +204,7 @@ async fn main() -> anyhow::Result<()> { ) } else { // `u16` cast is safe since i is in range [0, 4) - PrometheusExporterConfig::pull(prometheus_config.listener_port + i as u16) + PrometheusExporterConfig::pull(prometheus_listener_port + i as u16) }; let prometheus_task = prometheus_config.run(stop_receiver.clone()); From 585e612fb5bbeeb751dbd1bc351bd480bcd1f886 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:37:06 +0200 Subject: [PATCH 71/87] Remove mut --- prover/witness_generator/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 48fd5445c1af..ecb886f4cc4e 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -122,7 +122,7 @@ async fn main() -> anyhow::Result<()> { let config = general_config .witness_generator .context("witness generator config")?; - let mut prometheus_config = general_config + let prometheus_config = general_config .prometheus_config .context("prometheus config")?; From 0542ac0c0c32f4748fb0c2d38f6a67d28d303028 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:38:19 +0200 Subject: [PATCH 72/87] Fix prometheus_listener_port --- prover/witness_generator/src/main.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index ecb886f4cc4e..66dbe95b2387 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -128,7 +128,7 @@ async fn main() -> anyhow::Result<()> { // If the prometheus listener port is not set in the witness generator config, use the one from the prometheus config. let prometheus_listener_port = if let Some(port) = config.prometheus_listener_port { - port; + port } else { prometheus_config.listener_port }; From 5a4aaa2092e4470d08a7bee6e3291617fdb273ec Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Thu, 4 Jul 2024 16:41:00 +0200 Subject: [PATCH 73/87] Update prometheus listener ports --- etc/env/file_based/general.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index d50cf34ba957..861f5bd64454 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -19,7 +19,7 @@ db: api: prometheus: - listener_port: 3116 + listener_port: 3312 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 healthcheck: @@ -315,7 +315,7 @@ house_keeper: fri_gpu_prover_archiver_archive_after_secs: 172800 prometheus: - listener_port: 3116 + listener_port: 3313 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 From a7aec1866b971dd2712c6ec2ba6699c45e259af7 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 10:55:44 +0200 Subject: [PATCH 74/87] Do not fail if prometheus config not provided when not needed --- prover/witness_generator/src/main.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 66dbe95b2387..7380bc679112 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -122,14 +122,15 @@ async fn main() -> anyhow::Result<()> { let config = general_config .witness_generator .context("witness generator config")?; - let prometheus_config = general_config - .prometheus_config - .context("prometheus config")?; // If the prometheus listener port is not set in the witness generator config, use the one from the prometheus config. let prometheus_listener_port = if let Some(port) = config.prometheus_listener_port { port } else { + let prometheus_config = general_config + .prometheus_config + .clone() + .context("prometheus config")?; prometheus_config.listener_port }; @@ -198,6 +199,10 @@ async fn main() -> anyhow::Result<()> { ); let prometheus_config = if use_push_gateway { + let prometheus_config = general_config + .prometheus_config + .clone() + .context("prometheus config")?; PrometheusExporterConfig::push( prometheus_config.gateway_endpoint(), prometheus_config.push_interval(), From 2d2d09b10e40f4cfef8ac4af9cd09b4876ed0d91 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 10:59:21 +0200 Subject: [PATCH 75/87] Refactor prometheus config req --- prover/witness_generator/src/main.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index 7380bc679112..77d2887e152c 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -123,14 +123,13 @@ async fn main() -> anyhow::Result<()> { .witness_generator .context("witness generator config")?; + let prometheus_config = general_config.prometheus_config; + // If the prometheus listener port is not set in the witness generator config, use the one from the prometheus config. let prometheus_listener_port = if let Some(port) = config.prometheus_listener_port { port } else { - let prometheus_config = general_config - .prometheus_config - .clone() - .context("prometheus config")?; + let prometheus_config = prometheus_config.clone().context("prometheus config")?; prometheus_config.listener_port }; @@ -199,10 +198,9 @@ async fn main() -> anyhow::Result<()> { ); let prometheus_config = if use_push_gateway { - let prometheus_config = general_config - .prometheus_config + let prometheus_config = prometheus_config .clone() - .context("prometheus config")?; + .context("prometheus config needed when use_push_gateway enabled")?; PrometheusExporterConfig::push( prometheus_config.gateway_endpoint(), prometheus_config.push_interval(), From cd83f5a1fed958c781e0c84e7e38426477eee569 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 11:06:20 +0200 Subject: [PATCH 76/87] Remove unused variable --- prover/witness_generator/src/main.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/prover/witness_generator/src/main.rs b/prover/witness_generator/src/main.rs index f7a857f2e998..9f1c8d72cd1b 100644 --- a/prover/witness_generator/src/main.rs +++ b/prover/witness_generator/src/main.rs @@ -128,8 +128,10 @@ async fn main() -> anyhow::Result<()> { let prometheus_listener_port = if let Some(port) = config.prometheus_listener_port { port } else { - let prometheus_config = prometheus_config.clone().context("prometheus config")?; - prometheus_config.listener_port + prometheus_config + .clone() + .context("prometheus config")? + .listener_port }; let prover_connection_pool = From 324db127392517a69258c8f30744b8ec8282cf58 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 11:37:01 +0200 Subject: [PATCH 77/87] Update prometheus listener_port --- etc/env/file_based/general.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index 861f5bd64454..bf2d57125a57 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -315,7 +315,7 @@ house_keeper: fri_gpu_prover_archiver_archive_after_secs: 172800 prometheus: - listener_port: 3313 + listener_port: 3314 pushgateway_url: http://127.0.0.1:9091 push_interval_ms: 100 From 0c66acb069cd512cd380e2116aeab47f73268e20 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 11:44:33 +0200 Subject: [PATCH 78/87] Add nvidia-smi prerequisite --- zk_toolbox/crates/common/src/prerequisites.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/common/src/prerequisites.rs b/zk_toolbox/crates/common/src/prerequisites.rs index 8b101e86dc0c..6c437302470d 100644 --- a/zk_toolbox/crates/common/src/prerequisites.rs +++ b/zk_toolbox/crates/common/src/prerequisites.rs @@ -30,7 +30,7 @@ const DOCKER_COMPOSE_PREREQUISITE: Prerequisite = Prerequisite { download_link: "https://docs.docker.com/compose/install/", }; -const PROVER_PREREQUISITES: [Prerequisite; 4] = [ +const PROVER_PREREQUISITES: [Prerequisite; 5] = [ Prerequisite { name: "gcloud", download_link: "https://cloud.google.com/sdk/docs/install", @@ -46,7 +46,11 @@ const PROVER_PREREQUISITES: [Prerequisite; 4] = [ Prerequisite { name: "nvcc", download_link: "https://developer.nvidia.com/cuda-downloads", - }, + }, // CUDA toolkit + Prerequisite { + name: "nvidia-smi", + download_link: "https://developer.nvidia.com/cuda-downloads", + }, // CUDA GPU driver ]; struct Prerequisite { From 2ad428d2e36e9807c132c5181019ce3230256328 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 12:09:44 +0200 Subject: [PATCH 79/87] Fix setup key download --- zk_toolbox/crates/zk_inception/src/commands/prover/init.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 4797f81d2fb8..a55877e5ab99 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -88,9 +88,13 @@ fn download_setup_key( let url = compressor_config.universal_setup_download_url; let path = std::path::Path::new(path); let parent = path.parent().expect(MSG_SETUP_KEY_PATH_ERROR); + let file_name = path.file_name().expect(MSG_SETUP_KEY_PATH_ERROR); Cmd::new(cmd!(shell, "wget {url} -P {parent}")).run()?; - Cmd::new(cmd!(shell, "mv {parent}/setup_2^24.key {path}")).run()?; + + if file_name != "setup_2^24.key" { + Cmd::new(cmd!(shell, "mv {parent}/setup_2^24.key {path}")).run()?; + } spinner.finish(); Ok(()) From eeecf5a87ef1190c318f5a87655c3f910b3b90de Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 12:43:02 +0200 Subject: [PATCH 80/87] Fix prover gateway prometheus port --- etc/env/file_based/general.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index a75f3a98af5f..dd2e08d085ba 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -197,7 +197,7 @@ data_handler: prover_gateway: api_url: http://127.0.0.1:3320 api_poll_duration_secs: 1000 - prometheus_listener_port: 3314 + prometheus_listener_port: 3310 prometheus_pushgateway_url: http://127.0.0.1:9091 prometheus_push_interval_ms: 100 proof_compressor: From d08d76c21ea8f519d0a6a556fc2d80e7892b1184 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 15:54:13 +0200 Subject: [PATCH 81/87] Add InitBellmanCuda --- ZkStack.yaml | 1 + zk_toolbox/crates/config/src/ecosystem.rs | 12 +++++ .../src/commands/ecosystem/create.rs | 2 + .../commands/prover/args/init_bellman_cuda.rs | 29 +++++++++++ .../src/commands/prover/args/mod.rs | 1 + .../src/commands/prover/generate_sk.rs | 4 +- .../zk_inception/src/commands/prover/init.rs | 31 +----------- .../src/commands/prover/init_bellman_cuda.rs | 50 +++++++++++++++++++ .../zk_inception/src/commands/prover/mod.rs | 6 ++- .../zk_inception/src/commands/prover/run.rs | 35 ++++++------- .../crates/zk_inception/src/messages.rs | 4 ++ 11 files changed, 124 insertions(+), 51 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs diff --git a/ZkStack.yaml b/ZkStack.yaml index 33af09572190..68e9e10cd293 100644 --- a/ZkStack.yaml +++ b/ZkStack.yaml @@ -1,6 +1,7 @@ name: zk l1_network: Localhost link_to_code: . +bellman_cuda_dir: . chains: ./chains config: ./configs/ default_chain: era diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 08708ebb0b61..9c27227e99dc 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -25,6 +25,7 @@ struct EcosystemConfigInternal { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, + pub bellman_cuda_dir: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -40,6 +41,7 @@ pub struct EcosystemConfig { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, + pub bellman_cuda_dir: PathBuf, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -72,6 +74,11 @@ impl<'de> Deserialize<'de> for EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .to_path_buf(), + bellman_cuda_dir: config + .bellman_cuda_dir + .absolutize() + .expect("Failed to parse bellman-cuda path") + .to_path_buf(), chains: config.chains.clone(), config: config.config.clone(), default_chain: config.default_chain.clone(), @@ -202,6 +209,11 @@ impl EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .into(), + bellman_cuda_dir: self + .bellman_cuda_dir + .absolutize() + .expect("Failed to parse bellman-cuda path") + .into(), chains: self.chains.clone(), config: self.config.clone(), default_chain: self.default_chain.clone(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index 4daab36c56b8..2205cd7414b5 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -21,6 +21,7 @@ use crate::{ create_configs::{create_erc20_deployment_config, create_initial_deployments_config}, }, }, + consts::BELLMAN_CUDA_DIR, messages::{ MSG_CLONING_ERA_REPO_SPINNER, MSG_CREATED_ECOSYSTEM, MSG_CREATING_DEFAULT_CHAIN_SPINNER, MSG_CREATING_ECOSYSTEM, MSG_CREATING_INITIAL_CONFIGURATIONS_SPINNER, @@ -76,6 +77,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { name: ecosystem_name.clone(), l1_network: args.l1_network, link_to_code: link_to_code.clone(), + bellman_cuda_dir: shell.current_dir().join(BELLMAN_CUDA_DIR), chains: chains_path.clone(), config: configs_path, era_chain_id: get_default_era_chain_id(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs new file mode 100644 index 000000000000..c87795081625 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs @@ -0,0 +1,29 @@ +use std::path::PathBuf; + +use clap::Parser; +use common::Prompt; + +use crate::messages::MSG_BELLMAN_CUDA_DIR_PROMPT; + +#[derive(Debug, Clone, Parser, Default)] +pub struct InitBellmanCudaArgs { + #[clap(long)] + pub bellman_cuda_dir: Option, +} + +impl InitBellmanCudaArgs { + pub fn fill_values_with_prompt( + self, + default_bellman_cuda_dir: &PathBuf, + ) -> anyhow::Result { + let bellman_cuda_dir = self.bellman_cuda_dir.unwrap_or_else(|| { + Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) + .default(default_bellman_cuda_dir.to_str().unwrap_or_default()) + .ask() + }); + + Ok(InitBellmanCudaArgs { + bellman_cuda_dir: Some(bellman_cuda_dir), + }) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs index 0d06305e61dc..66d97d75094c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/mod.rs @@ -1,2 +1,3 @@ pub mod init; +pub mod init_bellman_cuda; pub mod run; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs index 7a92f193f9b8..1657ab2c99fb 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs @@ -1,5 +1,5 @@ use anyhow::Ok; -use common::{cmd::Cmd, logger, spinner::Spinner}; +use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; @@ -7,6 +7,8 @@ use super::utils::get_link_to_prover; use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { + check_prover_prequisites(shell); + let ecosystem_config = EcosystemConfig::from_file(shell)?; let link_to_prover = get_link_to_prover(&ecosystem_config); shell.change_dir(&link_to_prover); diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index a55877e5ab99..d8c4d192452f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -12,10 +12,9 @@ use super::{ utils::get_link_to_prover, }; use crate::{ - consts::{BELLMAN_CUDA_DIR, PROVER_STORE_MAX_RETRIES}, + consts::PROVER_STORE_MAX_RETRIES, messages::{ - MSG_BUILDING_BELLMAN_CUDA_SPINNER, MSG_CHAIN_NOT_FOUND_ERR, - MSG_CLONING_BELLMAN_CUDA_SPINNER, MSG_DOWNLOADING_SETUP_KEY_SPINNER, + MSG_CHAIN_NOT_FOUND_ERR, MSG_DOWNLOADING_SETUP_KEY_SPINNER, MSG_GENERAL_CONFIG_NOT_FOUND_ERR, MSG_PROOF_COMPRESSOR_CONFIG_NOT_FOUND_ERR, MSG_PROVER_CONFIG_NOT_FOUND_ERR, MSG_PROVER_INITIALIZED, MSG_SETUP_KEY_PATH_ERROR, }, @@ -68,8 +67,6 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( chain_config.save_zksync_general_config(&general_config)?; - init_bellman_cuda(shell)?; - logger::outro(MSG_PROVER_INITIALIZED); Ok(()) } @@ -136,27 +133,3 @@ fn get_object_store_config( Ok(object_store) } - -fn init_bellman_cuda(shell: &Shell) -> anyhow::Result<()> { - if shell.path_exists(BELLMAN_CUDA_DIR) { - return Ok(()); - } - - let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); - Cmd::new(cmd!( - shell, - "git clone https://github.com/matter-labs/era-bellman-cuda" - )) - .run()?; - spinner.finish(); - - let spinner = Spinner::new(MSG_BUILDING_BELLMAN_CUDA_SPINNER); - Cmd::new(cmd!( - shell, - "cmake -Bera-bellman-cuda/build -Sera-bellman-cuda/ -DCMAKE_BUILD_TYPE=Release" - )) - .run()?; - Cmd::new(cmd!(shell, "cmake --build era-bellman-cuda/build")).run()?; - spinner.finish(); - Ok(()) -} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs new file mode 100644 index 000000000000..d94e36a1aa9d --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs @@ -0,0 +1,50 @@ +use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; +use config::{traits::SaveConfigWithBasePath, EcosystemConfig}; +use xshell::{cmd, Shell}; + +use crate::messages::{ + MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED_MSG, MSG_BUILDING_BELLMAN_CUDA_SPINNER, + MSG_CLONING_BELLMAN_CUDA_SPINNER, +}; + +use super::args::init_bellman_cuda::InitBellmanCudaArgs; + +pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Result<()> { + check_prover_prequisites(shell); + + let mut ecosystem_config = EcosystemConfig::from_file(shell)?; + + let args = args.fill_values_with_prompt(&ecosystem_config.bellman_cuda_dir)?; + + let bellman_cuda_dir = args.bellman_cuda_dir.expect(MSG_BELLMAN_CUDA_DIR_ERR); + ecosystem_config.bellman_cuda_dir = bellman_cuda_dir.clone().into(); + + if shell.path_exists(&bellman_cuda_dir) { + return save_bellman_cuda_dir(shell, &ecosystem_config); + } + + let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); + Cmd::new(cmd!( + shell, + "git clone https://github.com/matter-labs/era-bellman-cuda {bellman_cuda_dir}" + )) + .run()?; + spinner.finish(); + + let spinner = Spinner::new(MSG_BUILDING_BELLMAN_CUDA_SPINNER); + Cmd::new(cmd!( + shell, + "cmake -B{bellman_cuda_dir}/build -S{bellman_cuda_dir}/ -DCMAKE_BUILD_TYPE=Release" + )) + .run()?; + Cmd::new(cmd!(shell, "cmake --build {bellman_cuda_dir}/build")).run()?; + spinner.finish(); + + save_bellman_cuda_dir(shell, &ecosystem_config) +} + +fn save_bellman_cuda_dir(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()> { + ecosystem.save_with_base_path(shell, ".")?; + logger::outro(MSG_BELLMAN_CUDA_INITIALIZED_MSG); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs index 94f173b591bd..d69e1e772e91 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -1,4 +1,4 @@ -use args::{init::ProverInitArgs, run::ProverRunArgs}; +use args::{init::ProverInitArgs, init_bellman_cuda::InitBellmanCudaArgs, run::ProverRunArgs}; use clap::Subcommand; use xshell::Shell; @@ -6,6 +6,7 @@ mod args; mod gcs; mod generate_sk; mod init; +mod init_bellman_cuda; mod run; mod utils; @@ -17,6 +18,8 @@ pub enum ProverCommands { GenerateSK, /// Run prover Run(ProverRunArgs), + /// Initialize bellman-cuda + InitBellmanCuda(Box), } pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { @@ -24,5 +27,6 @@ pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<( ProverCommands::Init(args) => init::run(*args, shell).await, ProverCommands::GenerateSK => generate_sk::run(shell).await, ProverCommands::Run(args) => run::run(args, shell).await, + ProverCommands::InitBellmanCuda(args) => init_bellman_cuda::run(shell, *args).await, } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index c5995a7a5826..39b6bec8b4c6 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -1,25 +1,18 @@ -use std::path::PathBuf; - use anyhow::Context; -use common::{cmd::Cmd, logger}; +use common::{check_prover_prequisites, cmd::Cmd, logger}; use config::{ChainConfig, EcosystemConfig}; -use path_absolutize::Absolutize; use xshell::{cmd, Shell}; use super::{ args::run::{ProverComponent, ProverRunArgs, WitnessGeneratorArgs, WitnessGeneratorRound}, utils::get_link_to_prover, }; -use crate::{ - consts::BELLMAN_CUDA_DIR, - messages::{ - MSG_BELLMAN_CUDA_DIR_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, - MSG_RUNNING_COMPRESSOR, MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, - MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, - MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, - MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, - MSG_WITNESS_GENERATOR_ROUND_ERR, - }, +use crate::messages::{ + MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, + MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR, + MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, + MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, + MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, MSG_WITNESS_GENERATOR_ROUND_ERR, }; pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<()> { @@ -41,7 +34,7 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() run_witness_vector_generator(shell, &chain)? } Some(ProverComponent::Prover) => run_prover(shell, &chain)?, - Some(ProverComponent::Compressor) => run_compressor(shell, &chain)?, + Some(ProverComponent::Compressor) => run_compressor(shell, &chain, &ecosystem_config)?, None => anyhow::bail!(MSG_MISSING_COMPONENT_ERR), } @@ -49,6 +42,7 @@ pub(crate) async fn run(args: ProverRunArgs, shell: &Shell) -> anyhow::Result<() } fn run_gateway(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { + check_prover_prequisites(shell); logger::info(MSG_RUNNING_PROVER_GATEWAY); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); @@ -104,15 +98,16 @@ fn run_prover(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { cmd.run().context(MSG_RUNNING_PROVER_ERR) } -fn run_compressor(shell: &Shell, chain: &ChainConfig) -> anyhow::Result<()> { +fn run_compressor( + shell: &Shell, + chain: &ChainConfig, + ecosystem: &EcosystemConfig, +) -> anyhow::Result<()> { logger::info(MSG_RUNNING_COMPRESSOR); let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - let bellman_cuda_dir = PathBuf::from(BELLMAN_CUDA_DIR); - let bellman_cuda_dir = bellman_cuda_dir.absolutize()?; - let bellman_cuda_str = bellman_cuda_dir.to_str().expect(MSG_BELLMAN_CUDA_DIR_ERR); - shell.set_var("BELLMAN_CUDA_DIR", bellman_cuda_str); + shell.set_var("BELLMAN_CUDA_DIR", ecosystem.bellman_cuda_dir.clone()); let mut cmd = Cmd::new(cmd!(shell, "cargo run --features gpu --release --bin zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 4eb0235c2298..0785a7a62a4a 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -263,6 +263,10 @@ pub(super) const MSG_SETUP_KEY_PATH_ERROR: &str = "Failed to get setup key path" pub(super) const MSG_CLONING_BELLMAN_CUDA_SPINNER: &str = "Cloning bellman-cuda..."; pub(super) const MSG_BUILDING_BELLMAN_CUDA_SPINNER: &str = "Building bellman-cuda..."; pub(super) const MSG_BELLMAN_CUDA_DIR_ERR: &str = "Failed to get bellman-cuda directory"; +pub(super) const MSG_BELLMAN_CUDA_DIR_PROMPT: &str = + "Provide the path to the bellman-cuda directory:"; +pub(super) const MSG_BELLMAN_CUDA_INITIALIZED_MSG: &str = + "bellman-cuda has been initialized successfully"; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 68cab6e1ead671689481870cf37103ef21900939 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 15:54:50 +0200 Subject: [PATCH 82/87] fmt --- .../zk_inception/src/commands/prover/init_bellman_cuda.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs index d94e36a1aa9d..c92ce49d7dcd 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs @@ -2,13 +2,12 @@ use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; use config::{traits::SaveConfigWithBasePath, EcosystemConfig}; use xshell::{cmd, Shell}; +use super::args::init_bellman_cuda::InitBellmanCudaArgs; use crate::messages::{ MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED_MSG, MSG_BUILDING_BELLMAN_CUDA_SPINNER, MSG_CLONING_BELLMAN_CUDA_SPINNER, }; -use super::args::init_bellman_cuda::InitBellmanCudaArgs; - pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Result<()> { check_prover_prequisites(shell); From 54f63430501180b7f766f8308106d873bfb856ef Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 16:10:19 +0200 Subject: [PATCH 83/87] lint --- .../src/commands/prover/args/init_bellman_cuda.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs index c87795081625..551ed8fa9c18 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs @@ -1,4 +1,4 @@ -use std::path::PathBuf; +use std::path::Path; use clap::Parser; use common::Prompt; @@ -14,7 +14,7 @@ pub struct InitBellmanCudaArgs { impl InitBellmanCudaArgs { pub fn fill_values_with_prompt( self, - default_bellman_cuda_dir: &PathBuf, + default_bellman_cuda_dir: &Path, ) -> anyhow::Result { let bellman_cuda_dir = self.bellman_cuda_dir.unwrap_or_else(|| { Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) From edce71a5b0b409d8e37ea0d78e45feb7bf78a8cc Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 16:43:25 +0200 Subject: [PATCH 84/87] Make bellman_cuda_dir optional --- ZkStack.yaml | 1 - zk_toolbox/crates/config/src/ecosystem.rs | 36 ++++++++++++------- .../src/commands/ecosystem/create.rs | 2 +- .../commands/prover/args/init_bellman_cuda.rs | 6 ++-- .../src/commands/prover/init_bellman_cuda.rs | 4 +-- .../zk_inception/src/commands/prover/run.rs | 12 +++++-- 6 files changed, 40 insertions(+), 21 deletions(-) diff --git a/ZkStack.yaml b/ZkStack.yaml index 68e9e10cd293..33af09572190 100644 --- a/ZkStack.yaml +++ b/ZkStack.yaml @@ -1,7 +1,6 @@ name: zk l1_network: Localhost link_to_code: . -bellman_cuda_dir: . chains: ./chains config: ./configs/ default_chain: era diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index 9c27227e99dc..cacdd3cd58d6 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -25,7 +25,7 @@ struct EcosystemConfigInternal { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, - pub bellman_cuda_dir: PathBuf, + pub bellman_cuda_dir: Option, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -41,7 +41,7 @@ pub struct EcosystemConfig { pub name: String, pub l1_network: L1Network, pub link_to_code: PathBuf, - pub bellman_cuda_dir: PathBuf, + pub bellman_cuda_dir: Option, pub chains: PathBuf, pub config: PathBuf, pub default_chain: String, @@ -66,6 +66,16 @@ impl<'de> Deserialize<'de> for EcosystemConfig { D: Deserializer<'de>, { let config: EcosystemConfigInternal = Deserialize::deserialize(deserializer)?; + let bellman_cuda_dir = if let Some(bellman_cuda_dir) = config.bellman_cuda_dir { + Some( + bellman_cuda_dir + .absolutize() + .expect("Failed to parse bellman-cuda path") + .to_path_buf(), + ) + } else { + None + }; Ok(EcosystemConfig { name: config.name.clone(), l1_network: config.l1_network, @@ -74,11 +84,7 @@ impl<'de> Deserialize<'de> for EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .to_path_buf(), - bellman_cuda_dir: config - .bellman_cuda_dir - .absolutize() - .expect("Failed to parse bellman-cuda path") - .to_path_buf(), + bellman_cuda_dir, chains: config.chains.clone(), config: config.config.clone(), default_chain: config.default_chain.clone(), @@ -201,6 +207,16 @@ impl EcosystemConfig { } fn get_internal(&self) -> EcosystemConfigInternal { + let bellman_cuda_dir = if let Some(bellman_cuda_dir) = self.bellman_cuda_dir.clone() { + Some( + bellman_cuda_dir + .absolutize() + .expect("Failed to parse bellman-cuda path") + .to_path_buf(), + ) + } else { + None + }; EcosystemConfigInternal { name: self.name.clone(), l1_network: self.l1_network, @@ -209,11 +225,7 @@ impl EcosystemConfig { .absolutize() .expect("Failed to parse zksync-era path") .into(), - bellman_cuda_dir: self - .bellman_cuda_dir - .absolutize() - .expect("Failed to parse bellman-cuda path") - .into(), + bellman_cuda_dir, chains: self.chains.clone(), config: self.config.clone(), default_chain: self.default_chain.clone(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index 2205cd7414b5..bdcf2fa2ad1d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -77,7 +77,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { name: ecosystem_name.clone(), l1_network: args.l1_network, link_to_code: link_to_code.clone(), - bellman_cuda_dir: shell.current_dir().join(BELLMAN_CUDA_DIR), + bellman_cuda_dir: Some(shell.current_dir().join(BELLMAN_CUDA_DIR)), chains: chains_path.clone(), config: configs_path, era_chain_id: get_default_era_chain_id(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs index 551ed8fa9c18..6b025e779461 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs @@ -1,4 +1,4 @@ -use std::path::Path; +use std::path::PathBuf; use clap::Parser; use common::Prompt; @@ -14,9 +14,11 @@ pub struct InitBellmanCudaArgs { impl InitBellmanCudaArgs { pub fn fill_values_with_prompt( self, - default_bellman_cuda_dir: &Path, + default_bellman_cuda_dir: Option, ) -> anyhow::Result { let bellman_cuda_dir = self.bellman_cuda_dir.unwrap_or_else(|| { + let default_bellman_cuda_dir = default_bellman_cuda_dir.unwrap_or(PathBuf::new()); + Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) .default(default_bellman_cuda_dir.to_str().unwrap_or_default()) .ask() diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs index c92ce49d7dcd..7dbd8a9bd5a2 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs @@ -13,10 +13,10 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res let mut ecosystem_config = EcosystemConfig::from_file(shell)?; - let args = args.fill_values_with_prompt(&ecosystem_config.bellman_cuda_dir)?; + let args = args.fill_values_with_prompt(ecosystem_config.bellman_cuda_dir)?; let bellman_cuda_dir = args.bellman_cuda_dir.expect(MSG_BELLMAN_CUDA_DIR_ERR); - ecosystem_config.bellman_cuda_dir = bellman_cuda_dir.clone().into(); + ecosystem_config.bellman_cuda_dir = Some(bellman_cuda_dir.clone().into()); if shell.path_exists(&bellman_cuda_dir) { return save_bellman_cuda_dir(shell, &ecosystem_config); diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs index 39b6bec8b4c6..f91e992f1fdc 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/run.rs @@ -8,8 +8,8 @@ use super::{ utils::get_link_to_prover, }; use crate::messages::{ - MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, MSG_RUNNING_COMPRESSOR, - MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR, + MSG_BELLMAN_CUDA_DIR_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_MISSING_COMPONENT_ERR, + MSG_RUNNING_COMPRESSOR, MSG_RUNNING_COMPRESSOR_ERR, MSG_RUNNING_PROVER, MSG_RUNNING_PROVER_ERR, MSG_RUNNING_PROVER_GATEWAY, MSG_RUNNING_PROVER_GATEWAY_ERR, MSG_RUNNING_WITNESS_GENERATOR, MSG_RUNNING_WITNESS_GENERATOR_ERR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR, MSG_RUNNING_WITNESS_VECTOR_GENERATOR_ERR, MSG_WITNESS_GENERATOR_ROUND_ERR, @@ -107,7 +107,13 @@ fn run_compressor( let config_path = chain.path_to_general_config(); let secrets_path = chain.path_to_secrets_config(); - shell.set_var("BELLMAN_CUDA_DIR", ecosystem.bellman_cuda_dir.clone()); + shell.set_var( + "BELLMAN_CUDA_DIR", + ecosystem + .bellman_cuda_dir + .clone() + .expect(MSG_BELLMAN_CUDA_DIR_ERR), + ); let mut cmd = Cmd::new(cmd!(shell, "cargo run --features gpu --release --bin zksync_proof_fri_compressor -- --config-path={config_path} --secrets-path={secrets_path}")); cmd = cmd.with_force_run(); From 22a2f1964ddb401892c7eb779cd0f3d992909454 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Mon, 8 Jul 2024 17:53:57 +0200 Subject: [PATCH 85/87] lint --- zk_toolbox/crates/config/src/ecosystem.rs | 30 ++++++++--------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index cacdd3cd58d6..de709c14f239 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -66,16 +66,11 @@ impl<'de> Deserialize<'de> for EcosystemConfig { D: Deserializer<'de>, { let config: EcosystemConfigInternal = Deserialize::deserialize(deserializer)?; - let bellman_cuda_dir = if let Some(bellman_cuda_dir) = config.bellman_cuda_dir { - Some( - bellman_cuda_dir - .absolutize() - .expect("Failed to parse bellman-cuda path") - .to_path_buf(), - ) - } else { - None - }; + let bellman_cuda_dir = config.bellman_cuda_dir.map(|dir| { + dir.absolutize() + .expect("Failed to parse bellman-cuda path") + .to_path_buf() + }); Ok(EcosystemConfig { name: config.name.clone(), l1_network: config.l1_network, @@ -207,16 +202,11 @@ impl EcosystemConfig { } fn get_internal(&self) -> EcosystemConfigInternal { - let bellman_cuda_dir = if let Some(bellman_cuda_dir) = self.bellman_cuda_dir.clone() { - Some( - bellman_cuda_dir - .absolutize() - .expect("Failed to parse bellman-cuda path") - .to_path_buf(), - ) - } else { - None - }; + let bellman_cuda_dir = self.bellman_cuda_dir.clone().map(|dir| { + dir.absolutize() + .expect("Failed to parse bellman-cuda path") + .to_path_buf() + }); EcosystemConfigInternal { name: self.name.clone(), l1_network: self.l1_network, From 3d55d839fcc031321921e3f819c02f88bba2eac1 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 9 Jul 2024 11:37:43 +0200 Subject: [PATCH 86/87] Refactor init bellmancuda --- .../src/commands/prover/args/init.rs | 22 ++++++++++ .../commands/prover/args/init_bellman_cuda.rs | 42 +++++++++++++++---- .../zk_inception/src/commands/prover/init.rs | 6 ++- .../src/commands/prover/init_bellman_cuda.rs | 25 ++++++----- .../crates/zk_inception/src/messages.rs | 6 ++- 5 files changed, 82 insertions(+), 19 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs index dc320305152f..68c12e6cbb54 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs @@ -1,3 +1,5 @@ +use std::path::PathBuf; + use clap::{Parser, ValueEnum}; use common::{logger, Prompt, PromptConfirm, PromptSelect}; use serde::{Deserialize, Serialize}; @@ -5,6 +7,7 @@ use strum::IntoEnumIterator; use strum_macros::EnumIter; use xshell::Shell; +use super::init_bellman_cuda::InitBellmanCudaArgs; use crate::{ commands::prover::gcs::get_project_ids, consts::{DEFAULT_CREDENTIALS_FILE, DEFAULT_PROOF_STORE_DIR}, @@ -44,6 +47,11 @@ pub struct ProverInitArgs { #[serde(flatten)] pub public_create_gcs_bucket_config: PublicStorageGCSCreateBucketTmp, + // Bellman cuda + #[clap(flatten)] + #[serde(flatten)] + pub bellman_cuda_config: InitBellmanCudaArgs, + #[clap(flatten)] #[serde(flatten)] pub setup_key_config: SetupKeyConfigTmp, @@ -138,6 +146,7 @@ pub struct ProverInitArgsFinal { pub proof_store: ProofStorageConfig, pub public_store: Option, pub setup_key_config: SetupKeyConfig, + pub bellman_cuda_config: InitBellmanCudaArgs, } impl ProverInitArgs { @@ -145,14 +154,18 @@ impl ProverInitArgs { &self, shell: &Shell, setup_key_path: &str, + default_bellman_cuda_dir: Option, ) -> anyhow::Result { let proof_store = self.fill_proof_storage_values_with_prompt(shell)?; let public_store = self.fill_public_storage_values_with_prompt(shell)?; let setup_key_config = self.fill_setup_key_values_with_prompt(setup_key_path); + let bellman_cuda_config = + self.fill_bellman_cuda_values_with_prompt(default_bellman_cuda_dir)?; Ok(ProverInitArgsFinal { proof_store, public_store, setup_key_config, + bellman_cuda_config, }) } @@ -394,4 +407,13 @@ impl ProverInitArgs { credentials_file, }) } + + fn fill_bellman_cuda_values_with_prompt( + &self, + default_bellman_cuda_dir: Option, + ) -> anyhow::Result { + self.bellman_cuda_config + .clone() + .fill_values_with_prompt(default_bellman_cuda_dir) + } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs index 6b025e779461..79877f82d75c 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs @@ -1,16 +1,36 @@ use std::path::PathBuf; use clap::Parser; -use common::Prompt; +use common::{Prompt, PromptSelect}; +use serde::{Deserialize, Serialize}; +use strum::{EnumIter, IntoEnumIterator}; -use crate::messages::MSG_BELLMAN_CUDA_DIR_PROMPT; +use crate::messages::{ + MSG_BELLMAN_CUDA_DIR_PROMPT, MSG_BELLMAN_CUDA_ORIGIN_SELECT, MSG_BELLMAN_CUDA_SELECTION_CLONE, + MSG_BELLMAN_CUDA_SELECTION_PATH, +}; -#[derive(Debug, Clone, Parser, Default)] +#[derive(Debug, Clone, Parser, Default, Serialize, Deserialize)] pub struct InitBellmanCudaArgs { #[clap(long)] pub bellman_cuda_dir: Option, } +#[derive(Debug, Clone, EnumIter, PartialEq, Eq)] +enum BellmanCudaPathSelection { + Clone, + Path, +} + +impl std::fmt::Display for BellmanCudaPathSelection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BellmanCudaPathSelection::Clone => write!(f, "{MSG_BELLMAN_CUDA_SELECTION_CLONE}"), + BellmanCudaPathSelection::Path => write!(f, "{MSG_BELLMAN_CUDA_SELECTION_PATH}"), + } + } +} + impl InitBellmanCudaArgs { pub fn fill_values_with_prompt( self, @@ -18,10 +38,18 @@ impl InitBellmanCudaArgs { ) -> anyhow::Result { let bellman_cuda_dir = self.bellman_cuda_dir.unwrap_or_else(|| { let default_bellman_cuda_dir = default_bellman_cuda_dir.unwrap_or(PathBuf::new()); - - Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) - .default(default_bellman_cuda_dir.to_str().unwrap_or_default()) - .ask() + let default_bellman_cuda_dir = default_bellman_cuda_dir.to_str().unwrap_or_default(); + match PromptSelect::new( + MSG_BELLMAN_CUDA_ORIGIN_SELECT, + BellmanCudaPathSelection::iter(), + ) + .ask() + { + BellmanCudaPathSelection::Clone => default_bellman_cuda_dir.to_string(), + BellmanCudaPathSelection::Path => Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) + .default(default_bellman_cuda_dir) + .ask(), + } }); Ok(InitBellmanCudaArgs { diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index d8c4d192452f..8a7ff13fbbda 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -9,6 +9,7 @@ use zksync_config::{ use super::{ args::init::{ProofStorageConfig, ProverInitArgs}, gcs::create_gcs_bucket, + init_bellman_cuda::run as init_bellman_cuda, utils::get_link_to_prover, }; use crate::{ @@ -31,8 +32,9 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( .expect(MSG_GENERAL_CONFIG_NOT_FOUND_ERR); let setup_key_path = get_default_setup_key_path(&ecosystem_config)?; + let default_bellman_cuda_dir = ecosystem_config.bellman_cuda_dir; - let args = args.fill_values_with_prompt(shell, &setup_key_path)?; + let args = args.fill_values_with_prompt(shell, &setup_key_path, default_bellman_cuda_dir)?; let proof_object_store_config = get_object_store_config(shell, Some(args.proof_store))?; let public_object_store_config = get_object_store_config(shell, args.public_store)?; @@ -67,6 +69,8 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( chain_config.save_zksync_general_config(&general_config)?; + init_bellman_cuda(shell, args.bellman_cuda_config).await?; + logger::outro(MSG_PROVER_INITIALIZED); Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs index 7dbd8a9bd5a2..ec3357ac179b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs @@ -4,7 +4,7 @@ use xshell::{cmd, Shell}; use super::args::init_bellman_cuda::InitBellmanCudaArgs; use crate::messages::{ - MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED_MSG, MSG_BUILDING_BELLMAN_CUDA_SPINNER, + MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED, MSG_BUILDING_BELLMAN_CUDA_SPINNER, MSG_CLONING_BELLMAN_CUDA_SPINNER, }; @@ -18,10 +18,19 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res let bellman_cuda_dir = args.bellman_cuda_dir.expect(MSG_BELLMAN_CUDA_DIR_ERR); ecosystem_config.bellman_cuda_dir = Some(bellman_cuda_dir.clone().into()); - if shell.path_exists(&bellman_cuda_dir) { - return save_bellman_cuda_dir(shell, &ecosystem_config); + if !shell.path_exists(&bellman_cuda_dir) { + clone_bellman_cuda(shell, &bellman_cuda_dir)?; } + build_bellman_cuda(shell, &bellman_cuda_dir)?; + + ecosystem_config.save_with_base_path(shell, ".")?; + + logger::outro(MSG_BELLMAN_CUDA_INITIALIZED); + Ok(()) +} + +fn clone_bellman_cuda(shell: &Shell, bellman_cuda_dir: &str) -> anyhow::Result<()> { let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); Cmd::new(cmd!( shell, @@ -29,7 +38,10 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res )) .run()?; spinner.finish(); + Ok(()) +} +fn build_bellman_cuda(shell: &Shell, bellman_cuda_dir: &str) -> anyhow::Result<()> { let spinner = Spinner::new(MSG_BUILDING_BELLMAN_CUDA_SPINNER); Cmd::new(cmd!( shell, @@ -38,12 +50,5 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res .run()?; Cmd::new(cmd!(shell, "cmake --build {bellman_cuda_dir}/build")).run()?; spinner.finish(); - - save_bellman_cuda_dir(shell, &ecosystem_config) -} - -fn save_bellman_cuda_dir(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow::Result<()> { - ecosystem.save_with_base_path(shell, ".")?; - logger::outro(MSG_BELLMAN_CUDA_INITIALIZED_MSG); Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 0785a7a62a4a..7e27a9ac366d 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -265,8 +265,12 @@ pub(super) const MSG_BUILDING_BELLMAN_CUDA_SPINNER: &str = "Building bellman-cud pub(super) const MSG_BELLMAN_CUDA_DIR_ERR: &str = "Failed to get bellman-cuda directory"; pub(super) const MSG_BELLMAN_CUDA_DIR_PROMPT: &str = "Provide the path to the bellman-cuda directory:"; -pub(super) const MSG_BELLMAN_CUDA_INITIALIZED_MSG: &str = +pub(super) const MSG_BELLMAN_CUDA_INITIALIZED: &str = "bellman-cuda has been initialized successfully"; +pub(super) const MSG_BELLMAN_CUDA_ORIGIN_SELECT: &str = + "Select the origin of bellman-cuda repository"; +pub(super) const MSG_BELLMAN_CUDA_SELECTION_CLONE: &str = "Clone for me (recommended)"; +pub(super) const MSG_BELLMAN_CUDA_SELECTION_PATH: &str = "I have the code already"; pub(super) fn msg_bucket_created(bucket_name: &str) -> String { format!("Bucket created successfully with url: gs://{bucket_name}") From 71536852463835790ef303602a1ab228d87b4747 Mon Sep 17 00:00:00 2001 From: matias-gonz Date: Tue, 9 Jul 2024 12:37:41 +0200 Subject: [PATCH 87/87] Refactor init bellmancuda --- .../src/commands/ecosystem/create.rs | 3 +- .../src/commands/prover/args/init.rs | 15 ++------ .../commands/prover/args/init_bellman_cuda.rs | 15 ++------ .../zk_inception/src/commands/prover/init.rs | 8 ++--- .../src/commands/prover/init_bellman_cuda.rs | 36 ++++++++++++------- 5 files changed, 35 insertions(+), 42 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index bdcf2fa2ad1d..a94c189d2b2b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -21,7 +21,6 @@ use crate::{ create_configs::{create_erc20_deployment_config, create_initial_deployments_config}, }, }, - consts::BELLMAN_CUDA_DIR, messages::{ MSG_CLONING_ERA_REPO_SPINNER, MSG_CREATED_ECOSYSTEM, MSG_CREATING_DEFAULT_CHAIN_SPINNER, MSG_CREATING_ECOSYSTEM, MSG_CREATING_INITIAL_CONFIGURATIONS_SPINNER, @@ -77,7 +76,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { name: ecosystem_name.clone(), l1_network: args.l1_network, link_to_code: link_to_code.clone(), - bellman_cuda_dir: Some(shell.current_dir().join(BELLMAN_CUDA_DIR)), + bellman_cuda_dir: None, chains: chains_path.clone(), config: configs_path, era_chain_id: get_default_era_chain_id(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs index 68c12e6cbb54..c398b1852c61 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use clap::{Parser, ValueEnum}; use common::{logger, Prompt, PromptConfirm, PromptSelect}; use serde::{Deserialize, Serialize}; @@ -154,13 +152,11 @@ impl ProverInitArgs { &self, shell: &Shell, setup_key_path: &str, - default_bellman_cuda_dir: Option, ) -> anyhow::Result { let proof_store = self.fill_proof_storage_values_with_prompt(shell)?; let public_store = self.fill_public_storage_values_with_prompt(shell)?; let setup_key_config = self.fill_setup_key_values_with_prompt(setup_key_path); - let bellman_cuda_config = - self.fill_bellman_cuda_values_with_prompt(default_bellman_cuda_dir)?; + let bellman_cuda_config = self.fill_bellman_cuda_values_with_prompt()?; Ok(ProverInitArgsFinal { proof_store, public_store, @@ -408,12 +404,7 @@ impl ProverInitArgs { }) } - fn fill_bellman_cuda_values_with_prompt( - &self, - default_bellman_cuda_dir: Option, - ) -> anyhow::Result { - self.bellman_cuda_config - .clone() - .fill_values_with_prompt(default_bellman_cuda_dir) + fn fill_bellman_cuda_values_with_prompt(&self) -> anyhow::Result { + self.bellman_cuda_config.clone().fill_values_with_prompt() } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs index 79877f82d75c..848457c53271 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/args/init_bellman_cuda.rs @@ -1,5 +1,3 @@ -use std::path::PathBuf; - use clap::Parser; use common::{Prompt, PromptSelect}; use serde::{Deserialize, Serialize}; @@ -32,23 +30,16 @@ impl std::fmt::Display for BellmanCudaPathSelection { } impl InitBellmanCudaArgs { - pub fn fill_values_with_prompt( - self, - default_bellman_cuda_dir: Option, - ) -> anyhow::Result { + pub fn fill_values_with_prompt(self) -> anyhow::Result { let bellman_cuda_dir = self.bellman_cuda_dir.unwrap_or_else(|| { - let default_bellman_cuda_dir = default_bellman_cuda_dir.unwrap_or(PathBuf::new()); - let default_bellman_cuda_dir = default_bellman_cuda_dir.to_str().unwrap_or_default(); match PromptSelect::new( MSG_BELLMAN_CUDA_ORIGIN_SELECT, BellmanCudaPathSelection::iter(), ) .ask() { - BellmanCudaPathSelection::Clone => default_bellman_cuda_dir.to_string(), - BellmanCudaPathSelection::Path => Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT) - .default(default_bellman_cuda_dir) - .ask(), + BellmanCudaPathSelection::Clone => "".to_string(), + BellmanCudaPathSelection::Path => Prompt::new(MSG_BELLMAN_CUDA_DIR_PROMPT).ask(), } }); diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs index 8a7ff13fbbda..31785338bf3e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; use config::EcosystemConfig; use xshell::{cmd, Shell}; @@ -26,15 +27,14 @@ pub(crate) async fn run(args: ProverInitArgs, shell: &Shell) -> anyhow::Result<( let ecosystem_config = EcosystemConfig::from_file(shell)?; let chain_config = ecosystem_config .load_chain(Some(ecosystem_config.default_chain.clone())) - .expect(MSG_CHAIN_NOT_FOUND_ERR); + .context(MSG_CHAIN_NOT_FOUND_ERR)?; let mut general_config = chain_config .get_zksync_general_config() - .expect(MSG_GENERAL_CONFIG_NOT_FOUND_ERR); + .context(MSG_GENERAL_CONFIG_NOT_FOUND_ERR)?; let setup_key_path = get_default_setup_key_path(&ecosystem_config)?; - let default_bellman_cuda_dir = ecosystem_config.bellman_cuda_dir; - let args = args.fill_values_with_prompt(shell, &setup_key_path, default_bellman_cuda_dir)?; + let args = args.fill_values_with_prompt(shell, &setup_key_path)?; let proof_object_store_config = get_object_store_config(shell, Some(args.proof_store))?; let public_object_store_config = get_object_store_config(shell, args.public_store)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs index ec3357ac179b..fd8efcd6eeb8 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/init_bellman_cuda.rs @@ -1,11 +1,15 @@ +use anyhow::Context; use common::{check_prover_prequisites, cmd::Cmd, logger, spinner::Spinner}; use config::{traits::SaveConfigWithBasePath, EcosystemConfig}; use xshell::{cmd, Shell}; use super::args::init_bellman_cuda::InitBellmanCudaArgs; -use crate::messages::{ - MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED, MSG_BUILDING_BELLMAN_CUDA_SPINNER, - MSG_CLONING_BELLMAN_CUDA_SPINNER, +use crate::{ + consts::BELLMAN_CUDA_DIR, + messages::{ + MSG_BELLMAN_CUDA_DIR_ERR, MSG_BELLMAN_CUDA_INITIALIZED, MSG_BUILDING_BELLMAN_CUDA_SPINNER, + MSG_CLONING_BELLMAN_CUDA_SPINNER, + }, }; pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Result<()> { @@ -13,14 +17,16 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res let mut ecosystem_config = EcosystemConfig::from_file(shell)?; - let args = args.fill_values_with_prompt(ecosystem_config.bellman_cuda_dir)?; + let args = args.fill_values_with_prompt()?; - let bellman_cuda_dir = args.bellman_cuda_dir.expect(MSG_BELLMAN_CUDA_DIR_ERR); - ecosystem_config.bellman_cuda_dir = Some(bellman_cuda_dir.clone().into()); + let bellman_cuda_dir = args.bellman_cuda_dir.unwrap_or("".to_string()); + let bellman_cuda_dir = if bellman_cuda_dir.is_empty() { + clone_bellman_cuda(shell)? + } else { + bellman_cuda_dir + }; - if !shell.path_exists(&bellman_cuda_dir) { - clone_bellman_cuda(shell, &bellman_cuda_dir)?; - } + ecosystem_config.bellman_cuda_dir = Some(bellman_cuda_dir.clone().into()); build_bellman_cuda(shell, &bellman_cuda_dir)?; @@ -30,15 +36,21 @@ pub(crate) async fn run(shell: &Shell, args: InitBellmanCudaArgs) -> anyhow::Res Ok(()) } -fn clone_bellman_cuda(shell: &Shell, bellman_cuda_dir: &str) -> anyhow::Result<()> { +fn clone_bellman_cuda(shell: &Shell) -> anyhow::Result { let spinner = Spinner::new(MSG_CLONING_BELLMAN_CUDA_SPINNER); Cmd::new(cmd!( shell, - "git clone https://github.com/matter-labs/era-bellman-cuda {bellman_cuda_dir}" + "git clone https://github.com/matter-labs/era-bellman-cuda" )) .run()?; spinner.finish(); - Ok(()) + + Ok(shell + .current_dir() + .join(BELLMAN_CUDA_DIR) + .to_str() + .context(MSG_BELLMAN_CUDA_DIR_ERR)? + .to_string()) } fn build_bellman_cuda(shell: &Shell, bellman_cuda_dir: &str) -> anyhow::Result<()> {