feat(zk_toolbox): Migrate CI unit tests to zk_toolbox

.github/workflows/ci-core-reusable.yml

@@ -58,13 +58,14 @@ jobs:
           ci_run zk db setup
           ci_run zk compiler all
           ci_run zk contract build
+          ci_run ./bin/zkt
 
       - name: Contracts unit tests
         run: ci_run yarn l1-contracts test
 
       - name: Rust unit tests
         run: |
-          ci_run zk test rust
+          ci_run zk_supervisor test rust --link-to-code=/usr/src/zksync --test-server-url=postgres://postgres:notsecurepassword@localhost:5433/zksync_local_test --test-prover-url=postgres://postgres:notsecurepassword@localhost:5433/prover_local_test
           # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
           # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
           ci_run zk f cargo test --release -p vm-benchmark --bench oneshot --bench batch

core/node/external_proof_integration_api/src/lib.rs

@@ -4,8 +4,6 @@ mod middleware;
 mod processor;
 mod types;
 
-pub use crate::processor::Processor;
-
 use std::net::SocketAddr;
 
 use anyhow::Context;
@@ -20,6 +18,7 @@ use tokio::sync::watch;
 use types::{ExternalProof, ProofGenerationDataResponse};
 use zksync_basic_types::L1BatchNumber;
 
+pub use crate::processor::Processor;
 use crate::{
     metrics::{CallOutcome, Method},
     middleware::MetricsMiddleware,

zk_toolbox/crates/zk_supervisor/src/commands/test/args/rust.rs

@@ -1,9 +1,18 @@
 use clap::Parser;
 
-use crate::messages::MSG_TEST_RUST_OPTIONS_HELP;
+use crate::messages::{
+    MSG_TEST_PROVER_URL_HELP, MSG_TEST_RUST_LINK_TO_CODE_HELP, MSG_TEST_RUST_OPTIONS_HELP,
+    MSG_TEST_SERVER_URL_HELP,
+};
 
 #[derive(Debug, Parser)]
 pub struct RustArgs {
     #[clap(long, help = MSG_TEST_RUST_OPTIONS_HELP)]
     pub options: Option<String>,
+    #[clap(long, help = MSG_TEST_RUST_LINK_TO_CODE_HELP)]
+    pub link_to_code: Option<String>,
+    #[clap(long, help = MSG_TEST_SERVER_URL_HELP)]
+    pub test_server_url: Option<String>,
+    #[clap(long, help = MSG_TEST_PROVER_URL_HELP)]
+    pub test_prover_url: Option<String>,
 }

zk_toolbox/crates/zk_supervisor/src/commands/test/rust.rs

@@ -1,32 +1,73 @@
+use std::{
+    path::{Path, PathBuf},
+    str::FromStr,
+};
+
 use anyhow::Context;
 use common::{cmd::Cmd, config::global_config, db::wait_for_db, logger};
 use config::EcosystemConfig;
+use url::Url;
 use xshell::{cmd, Shell};
 
 use super::args::rust::RustArgs;
 use crate::{
     commands::database,
-    dals::get_test_dals,
+    dals::{Dal, CORE_DAL_PATH, PROVER_DAL_PATH},
     messages::{
         MSG_CARGO_NEXTEST_MISSING_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_POSTGRES_CONFIG_NOT_FOUND_ERR,
         MSG_RESETTING_TEST_DATABASES, MSG_UNIT_TESTS_RUN_SUCCESS, MSG_USING_CARGO_NEXTEST,
     },
 };
 
 pub async fn run(shell: &Shell, args: RustArgs) -> anyhow::Result<()> {
-    let ecosystem = EcosystemConfig::from_file(shell)?;
-    let chain = ecosystem
-        .clone()
-        .load_chain(global_config().chain_name.clone())
-        .context(MSG_CHAIN_NOT_FOUND_ERR)?;
-    let general_config = chain.get_general_config()?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?;
+    let link_to_code = if let Some(link_to_code) = args.link_to_code {
+        PathBuf::from(link_to_code)
+    } else {
+        EcosystemConfig::from_file(shell)?.link_to_code
+    };
+
+    let (test_server_url, test_prover_url) =
+        if args.test_prover_url.is_none() || args.test_server_url.is_none() {
+            let ecosystem = EcosystemConfig::from_file(shell)?;
+            let chain = ecosystem
+                .clone()
+                .load_chain(Some(ecosystem.default_chain))
+                .context(MSG_CHAIN_NOT_FOUND_ERR)?;
+            let general_config = chain.get_general_config()?;
+            let postgres = general_config
+                .postgres_config
+                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?;
+
+            (
+                args.test_server_url.unwrap_or(
+                    postgres
+                        .test_server_url
+                        .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
+                ),
+                args.test_prover_url.unwrap_or(
+                    postgres
+                        .test_prover_url
+                        .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
+                ),
+            )
+        } else {
+            (args.test_server_url.unwrap(), args.test_prover_url.unwrap())
+        };
+
+    let dals = vec![
+        Dal {
+            url: Url::from_str(&test_server_url.clone())?,
+            path: CORE_DAL_PATH.to_string(),
+        },
+        Dal {
+            url: Url::from_str(&test_prover_url.clone())?,
+            path: PROVER_DAL_PATH.to_string(),
+        },
+    ];
 
-    reset_test_databases(shell).await?;
+    reset_test_databases(shell, &link_to_code, dals).await?;
 
-    let _dir_guard = shell.push_dir(&ecosystem.link_to_code);
+    let _dir_guard = shell.push_dir(&link_to_code);
 
     let cmd = if nextest_is_installed(shell)? {
         logger::info(MSG_USING_CARGO_NEXTEST);
@@ -43,18 +84,8 @@ pub async fn run(shell: &Shell, args: RustArgs) -> anyhow::Result<()> {
     };
 
     let cmd = cmd
-        .env(
-            "TEST_DATABASE_URL",
-            postgres
-                .test_server_url
-                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
-        )
-        .env(
-            "TEST_PROVER_DATABASE_URL",
-            postgres
-                .test_prover_url
-                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
-        );
+        .env("TEST_DATABASE_URL", test_server_url)
+        .env("TEST_PROVER_DATABASE_URL", test_prover_url);
     cmd.run()?;
 
     logger::outro(MSG_UNIT_TESTS_RUN_SUCCESS);
@@ -70,9 +101,12 @@ fn nextest_is_installed(shell: &Shell) -> anyhow::Result<bool> {
     Ok(out.contains("cargo-nextest"))
 }
 
-async fn reset_test_databases(shell: &Shell) -> anyhow::Result<()> {
+async fn reset_test_databases(
+    shell: &Shell,
+    link_to_code: &Path,
+    dals: Vec<Dal>,
+) -> anyhow::Result<()> {
     logger::info(MSG_RESETTING_TEST_DATABASES);
-    let ecosystem = EcosystemConfig::from_file(shell)?;
 
     Cmd::new(cmd!(
         shell,
@@ -85,11 +119,12 @@ async fn reset_test_databases(shell: &Shell) -> anyhow::Result<()> {
     ))
     .run()?;
 
-    for dal in get_test_dals(shell)? {
+    for dal in dals {
         let mut url = dal.url.clone();
         url.set_path("");
+        logger::debug(&format!("Waiting for database: {}", url));
         wait_for_db(&url, 3).await?;
-        database::reset::reset_database(shell, ecosystem.link_to_code.clone(), dal.clone()).await?;
+        database::reset::reset_database(shell, link_to_code, dal.clone()).await?;
     }
 
     Ok(())

zk_toolbox/crates/zk_supervisor/src/dals.rs

@@ -1,5 +1,3 @@
-use std::str::FromStr;
-
 use anyhow::{anyhow, Context};
 use common::config::global_config;
 use config::{EcosystemConfig, SecretsConfig};
@@ -11,8 +9,8 @@ use crate::{
     messages::{MSG_CHAIN_NOT_FOUND_ERR, MSG_DATABASE_MUST_BE_PRESENTED},
 };
 
-const CORE_DAL_PATH: &str = "core/lib/dal";
-const PROVER_DAL_PATH: &str = "prover/crates/lib/prover_dal";
+pub const CORE_DAL_PATH: &str = "core/lib/dal";
+pub const PROVER_DAL_PATH: &str = "prover/crates/lib/prover_dal";
 
 #[derive(Debug, Clone)]
 pub struct SelectedDals {
@@ -50,10 +48,6 @@ pub fn get_dals(
     Ok(dals)
 }
 
-pub fn get_test_dals(shell: &Shell) -> anyhow::Result<Vec<Dal>> {
-    Ok(vec![get_test_prover_dal(shell)?, get_test_core_dal(shell)?])
-}
-
 pub fn get_prover_dal(shell: &Shell, url: Option<String>) -> anyhow::Result<Dal> {
     let url = if let Some(url) = url {
         Url::parse(&url)?
@@ -94,51 +88,6 @@ pub fn get_core_dal(shell: &Shell, url: Option<String>) -> anyhow::Result<Dal> {
     })
 }
 
-pub fn get_test_core_dal(shell: &Shell) -> anyhow::Result<Dal> {
-    let general_config = get_general_config(shell)?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_DATABASE_MUST_BE_PRESENTED)?;
-
-    let url = Url::from_str(
-        &postgres
-            .test_server_url
-            .clone()
-            .context(MSG_DATABASE_MUST_BE_PRESENTED)?,
-    )?;
-    Ok(Dal {
-        path: CORE_DAL_PATH.to_string(),
-        url,
-    })
-}
-
-pub fn get_test_prover_dal(shell: &Shell) -> anyhow::Result<Dal> {
-    let general_config = get_general_config(shell)?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_DATABASE_MUST_BE_PRESENTED)?;
-
-    let url = Url::from_str(
-        &postgres
-            .test_prover_url
-            .clone()
-            .context(MSG_DATABASE_MUST_BE_PRESENTED)?,
-    )?;
-
-    Ok(Dal {
-        path: PROVER_DAL_PATH.to_string(),
-        url,
-    })
-}
-
-fn get_general_config(shell: &Shell) -> anyhow::Result<config::GeneralConfig> {
-    let ecosystem_config = EcosystemConfig::from_file(shell)?;
-    let chain_config = ecosystem_config
-        .load_chain(global_config().chain_name.clone())
-        .context(MSG_CHAIN_NOT_FOUND_ERR)?;
-    chain_config.get_general_config()
-}
-
 fn get_secrets(shell: &Shell) -> anyhow::Result<SecretsConfig> {
     let ecosystem_config = EcosystemConfig::from_file(shell)?;
     let chain_config = ecosystem_config

zk_toolbox/crates/zk_supervisor/src/messages.rs

@@ -88,6 +88,9 @@ pub(super) const MSG_RECOVERY_TEST_ABOUT: &str = "Run recovery tests";
 pub(super) const MSG_UPGRADE_TEST_ABOUT: &str = "Run upgrade tests";
 pub(super) const MSG_RUST_TEST_ABOUT: &str = "Run unit-tests, accepts optional cargo test flags";
 pub(super) const MSG_TEST_RUST_OPTIONS_HELP: &str = "Cargo test flags";
+pub(super) const MSG_TEST_RUST_LINK_TO_CODE_HELP: &str = "Link to code";
+pub(super) const MSG_TEST_SERVER_URL_HELP: &str = "Test database URL";
+pub(super) const MSG_TEST_PROVER_URL_HELP: &str = "Test prover database URL";
 pub(super) const MSG_BUILD_ABOUT: &str = "Build all test dependencies";
 pub(super) const MSG_TESTS_EXTERNAL_NODE_HELP: &str = "Run tests for external node";
 pub(super) const MSG_NO_DEPS_HELP: &str = "Do not install or build dependencies";