diff --git a/core/lib/env_config/src/chain.rs b/core/lib/env_config/src/chain.rs index f62f8b859caa..a25c593bd881 100644 --- a/core/lib/env_config/src/chain.rs +++ b/core/lib/env_config/src/chain.rs @@ -131,6 +131,7 @@ mod tests { CHAIN_STATE_KEEPER_BATCH_OVERHEAD_L1_GAS="800000" CHAIN_STATE_KEEPER_MAX_GAS_PER_BATCH="200000000" CHAIN_STATE_KEEPER_MAX_PUBDATA_PER_BATCH="100000" + CHAIN_STATE_KEEPER_MAX_CIRCUITS_PER_BATCH="24100" CHAIN_STATE_KEEPER_FEE_MODEL_VERSION="V2" CHAIN_STATE_KEEPER_VALIDATION_COMPUTATIONAL_GAS_LIMIT="10000000" CHAIN_STATE_KEEPER_SAVE_CALL_TRACES="false" diff --git a/core/lib/env_config/src/eth_sender.rs b/core/lib/env_config/src/eth_sender.rs index 30a6ebf4f008..64e0a89d5a42 100644 --- a/core/lib/env_config/src/eth_sender.rs +++ b/core/lib/env_config/src/eth_sender.rs @@ -134,6 +134,8 @@ mod tests { ETH_SENDER_SENDER_L1_BATCH_MIN_AGE_BEFORE_EXECUTE_SECONDS="1000" ETH_SENDER_SENDER_MAX_ACCEPTABLE_PRIORITY_FEE_IN_GWEI="100000000000" ETH_SENDER_SENDER_PUBDATA_SENDING_MODE="Calldata" + ETH_WATCH_CONFIRMATIONS_FOR_ETH_EVENT="0" + ETH_WATCH_ETH_NODE_POLL_INTERVAL="300" ETH_CLIENT_WEB3_URL="http://127.0.0.1:8545" "#; diff --git a/core/lib/env_config/src/house_keeper.rs b/core/lib/env_config/src/house_keeper.rs index f23d2705bd0b..25eeda793937 100644 --- a/core/lib/env_config/src/house_keeper.rs +++ b/core/lib/env_config/src/house_keeper.rs @@ -45,6 +45,7 @@ mod tests { HOUSE_KEEPER_PROVER_JOB_RETRYING_INTERVAL_MS="10000" HOUSE_KEEPER_WITNESS_JOB_MOVING_INTERVAL_MS="30000" HOUSE_KEEPER_WITNESS_GENERATOR_STATS_REPORTING_INTERVAL_MS="10000" + HOUSE_KEEPER_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" HOUSE_KEEPER_FRI_WITNESS_JOB_MOVING_INTERVAL_MS="40000" HOUSE_KEEPER_FRI_PROVER_JOB_RETRYING_INTERVAL_MS="30000" HOUSE_KEEPER_FRI_WITNESS_GENERATOR_JOB_RETRYING_INTERVAL_MS="30000" diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/database/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/database/mod.rs index 74c4063a6974..e942e6f3f4f8 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/database/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/database/mod.rs @@ -14,7 +14,7 @@ mod drop; mod migrate; mod new_migration; mod prepare; -mod reset; +pub mod reset; mod setup; #[derive(Subcommand, Debug)] diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/database/reset.rs b/zk_toolbox/crates/zk_supervisor/src/commands/database/reset.rs index aa813a155510..d25f2a8cd54b 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/database/reset.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/database/reset.rs @@ -35,7 +35,7 @@ pub async fn run(shell: &Shell, args: DatabaseCommonArgs) -> anyhow::Result<()> Ok(()) } -async fn reset_database( +pub async fn reset_database( shell: &Shell, link_to_code: impl AsRef, dal: Dal, diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs index fc6098488971..ddd5c5588a0c 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs @@ -1,3 +1,4 @@ pub mod integration; pub mod recovery; pub mod revert; +pub mod rust; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/rust.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/rust.rs new file mode 100644 index 000000000000..2d94adc3f6a7 --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/rust.rs @@ -0,0 +1,9 @@ +use clap::Parser; + +use crate::messages::MSG_TEST_RUST_OPTIONS_HELP; + +#[derive(Debug, Parser)] +pub struct RustArgs { + #[clap(long, help = MSG_TEST_RUST_OPTIONS_HELP)] + pub options: Option, +} diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs index b22189078da4..70177888d1d5 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs @@ -1,10 +1,12 @@ -use args::{integration::IntegrationArgs, recovery::RecoveryArgs, revert::RevertArgs}; +use args::{ + integration::IntegrationArgs, recovery::RecoveryArgs, revert::RevertArgs, rust::RustArgs, +}; use clap::Subcommand; use xshell::Shell; use crate::messages::{ MSG_INTEGRATION_TESTS_ABOUT, MSG_L1_CONTRACTS_ABOUT, MSG_PROVER_TEST_ABOUT, - MSG_RECOVERY_TEST_ABOUT, MSG_REVERT_TEST_ABOUT, MSG_UPGRADE_TEST_ABOUT, + MSG_RECOVERY_TEST_ABOUT, MSG_REVERT_TEST_ABOUT, MSG_RUST_TEST_ABOUT, MSG_UPGRADE_TEST_ABOUT, }; mod args; @@ -13,6 +15,7 @@ mod l1_contracts; mod prover; mod recovery; mod revert; +mod rust; mod upgrade; #[derive(Subcommand, Debug)] @@ -25,18 +28,21 @@ pub enum TestCommands { Recovery(RecoveryArgs), #[clap(about = MSG_UPGRADE_TEST_ABOUT, alias = "u")] Upgrade, + #[clap(about = MSG_RUST_TEST_ABOUT, alias = "unit")] + Rust(RustArgs), #[clap(about = MSG_L1_CONTRACTS_ABOUT, alias = "l1")] L1Contracts, #[clap(about = MSG_PROVER_TEST_ABOUT, alias = "p")] Prover, } -pub fn run(shell: &Shell, args: TestCommands) -> anyhow::Result<()> { +pub async fn run(shell: &Shell, args: TestCommands) -> anyhow::Result<()> { match args { TestCommands::Integration(args) => integration::run(shell, args), TestCommands::Revert(args) => revert::run(shell, args), TestCommands::Recovery(args) => recovery::run(shell, args), TestCommands::Upgrade => upgrade::run(shell), + TestCommands::Rust(args) => rust::run(shell, args).await, TestCommands::L1Contracts => l1_contracts::run(shell), TestCommands::Prover => prover::run(shell), } diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/rust.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/rust.rs new file mode 100644 index 000000000000..9134ad08246e --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/rust.rs @@ -0,0 +1,96 @@ +use anyhow::Context; +use common::{cmd::Cmd, db::wait_for_db, logger}; +use config::EcosystemConfig; +use xshell::{cmd, Shell}; + +use super::args::rust::RustArgs; +use crate::{ + commands::database, + dals::get_test_dals, + messages::{ + MSG_CARGO_NEXTEST_MISSING_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_POSTGRES_CONFIG_NOT_FOUND_ERR, + MSG_RESETTING_TEST_DATABASES, MSG_UNIT_TESTS_RUN_SUCCESS, MSG_USING_CARGO_NEXTEST, + }, +}; + +pub async fn run(shell: &Shell, args: RustArgs) -> anyhow::Result<()> { + let ecosystem = EcosystemConfig::from_file(shell)?; + let chain = ecosystem + .clone() + .load_chain(Some(ecosystem.default_chain)) + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + let general_config = chain.get_general_config()?; + let postgres = general_config + .postgres_config + .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?; + + reset_test_databases(shell).await?; + + let _dir_guard = shell.push_dir(&ecosystem.link_to_code); + + let cmd = if nextest_is_installed(shell)? { + logger::info(MSG_USING_CARGO_NEXTEST); + cmd!(shell, "cargo nextest run --release") + } else { + logger::error(MSG_CARGO_NEXTEST_MISSING_ERR); + cmd!(shell, "cargo test --release") + }; + + let cmd = if let Some(options) = args.options { + Cmd::new(cmd.args(options.split_whitespace())).with_force_run() + } else { + Cmd::new(cmd).with_force_run() + }; + + let cmd = cmd + .env( + "TEST_DATABASE_URL", + postgres + .test_server_url + .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?, + ) + .env( + "TEST_PROVER_DATABASE_URL", + postgres + .test_prover_url + .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?, + ); + cmd.run()?; + + logger::outro(MSG_UNIT_TESTS_RUN_SUCCESS); + Ok(()) +} + +fn nextest_is_installed(shell: &Shell) -> anyhow::Result { + let out = String::from_utf8( + Cmd::new(cmd!(shell, "cargo install --list")) + .run_with_output()? + .stdout, + )?; + Ok(out.contains("cargo-nextest")) +} + +async fn reset_test_databases(shell: &Shell) -> anyhow::Result<()> { + logger::info(MSG_RESETTING_TEST_DATABASES); + let ecosystem = EcosystemConfig::from_file(shell)?; + + Cmd::new(cmd!( + shell, + "docker compose -f docker-compose-unit-tests.yml down" + )) + .run()?; + Cmd::new(cmd!( + shell, + "docker compose -f docker-compose-unit-tests.yml up -d" + )) + .run()?; + + for dal in get_test_dals(shell)? { + let mut url = dal.url.clone(); + url.set_path(""); + wait_for_db(&url, 3).await?; + database::reset::reset_database(shell, ecosystem.link_to_code.clone(), dal.clone()).await?; + } + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_supervisor/src/dals.rs b/zk_toolbox/crates/zk_supervisor/src/dals.rs index 2d2af41500b4..8a68d443ef3d 100644 --- a/zk_toolbox/crates/zk_supervisor/src/dals.rs +++ b/zk_toolbox/crates/zk_supervisor/src/dals.rs @@ -1,3 +1,5 @@ +use std::str::FromStr; + use anyhow::{anyhow, Context}; use common::config::global_config; use config::{EcosystemConfig, SecretsConfig}; @@ -41,6 +43,10 @@ pub fn get_dals(shell: &Shell, selected_dals: &SelectedDals) -> anyhow::Result anyhow::Result> { + Ok(vec![get_test_prover_dal(shell)?, get_test_core_dal(shell)?]) +} + pub fn get_prover_dal(shell: &Shell) -> anyhow::Result { let secrets = get_secrets(shell)?; @@ -71,6 +77,51 @@ pub fn get_core_dal(shell: &Shell) -> anyhow::Result { }) } +pub fn get_test_core_dal(shell: &Shell) -> anyhow::Result { + let general_config = get_general_config(shell)?; + let postgres = general_config + .postgres_config + .context(MSG_DATABASE_MUST_BE_PRESENTED)?; + + let url = Url::from_str( + &postgres + .test_server_url + .clone() + .context(MSG_DATABASE_MUST_BE_PRESENTED)?, + )?; + Ok(Dal { + path: CORE_DAL_PATH.to_string(), + url, + }) +} + +pub fn get_test_prover_dal(shell: &Shell) -> anyhow::Result { + let general_config = get_general_config(shell)?; + let postgres = general_config + .postgres_config + .context(MSG_DATABASE_MUST_BE_PRESENTED)?; + + let url = Url::from_str( + &postgres + .test_prover_url + .clone() + .context(MSG_DATABASE_MUST_BE_PRESENTED)?, + )?; + + Ok(Dal { + path: PROVER_DAL_PATH.to_string(), + url, + }) +} + +fn get_general_config(shell: &Shell) -> anyhow::Result { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_chain(global_config().chain_name.clone()) + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + chain_config.get_general_config() +} + fn get_secrets(shell: &Shell) -> anyhow::Result { let ecosystem_config = EcosystemConfig::from_file(shell)?; let chain_config = ecosystem_config diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index 51b8f00ef373..6d2a6f2c0007 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -92,7 +92,7 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { match args.command { SupervisorSubcommands::Database(command) => commands::database::run(shell, command).await?, - SupervisorSubcommands::Test(command) => commands::test::run(shell, command)?, + SupervisorSubcommands::Test(command) => commands::test::run(shell, command).await?, SupervisorSubcommands::Clean(command) => commands::clean::run(shell, command)?, SupervisorSubcommands::Snapshot(command) => commands::snapshot::run(shell, command).await?, SupervisorSubcommands::Markdown => { diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 6368cb4e3d53..db370d13615f 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -77,13 +77,20 @@ pub(super) const MSG_INTEGRATION_TESTS_ABOUT: &str = "Run integration tests"; pub(super) const MSG_REVERT_TEST_ABOUT: &str = "Run revert tests"; pub(super) const MSG_RECOVERY_TEST_ABOUT: &str = "Run recovery tests"; pub(super) const MSG_UPGRADE_TEST_ABOUT: &str = "Run upgrade tests"; +pub(super) const MSG_RUST_TEST_ABOUT: &str = "Run unit-tests, accepts optional cargo test flags"; +pub(super) const MSG_TEST_RUST_OPTIONS_HELP: &str = "Cargo test flags"; pub(super) const MSG_TESTS_EXTERNAL_NODE_HELP: &str = "Run tests for external node"; pub(super) const MSG_TESTS_RECOVERY_SNAPSHOT_HELP: &str = "Run recovery from a snapshot instead of genesis"; +pub(super) const MSG_UNIT_TESTS_RUN_SUCCESS: &str = "Unit tests ran successfully"; +pub(super) const MSG_USING_CARGO_NEXTEST: &str = "Using cargo-nextest for running tests"; +pub(super) const MSG_CARGO_NEXTEST_MISSING_ERR: &str = "cargo-nextest is missing, please run 'cargo install cargo-nextest'. Falling back to 'cargo test'"; pub(super) const MSG_L1_CONTRACTS_ABOUT: &str = "Run L1 contracts tests"; pub(super) const MSG_L1_CONTRACTS_TEST_SUCCESS: &str = "L1 contracts tests ran successfully"; pub(super) const MSG_PROVER_TEST_ABOUT: &str = "Run prover tests"; pub(super) const MSG_PROVER_TEST_SUCCESS: &str = "Prover tests ran successfully"; +pub(super) const MSG_POSTGRES_CONFIG_NOT_FOUND_ERR: &str = "Postgres config not found"; +pub(super) const MSG_RESETTING_TEST_DATABASES: &str = "Resetting test databases"; // Integration tests related messages pub(super) fn msg_integration_tests_run(external_node: bool) -> String {