feat(zk_toolbox): Migrate CI unit tests to zk_toolbox (#2759)

## What ❔
Migrate CI unit tests to zk_toolbox

.github/workflows/ci-core-reusable.yml

@@ -52,22 +52,19 @@ jobs:
 
       - name: Init
         run: |
-          ci_run zk
           ci_run run_retried rustup show
-          ci_run zk run yarn
-          ci_run zk db setup
-          ci_run zk compiler all
-          ci_run zk contract build
+          ci_run ./bin/zkt
+          ci_run zk_supervisor contracts
 
       - name: Contracts unit tests
         run: ci_run yarn l1-contracts test
 
       - name: Rust unit tests
         run: |
-          ci_run zk test rust
+          ci_run zk_supervisor test rust
           # Benchmarks are not tested by `cargo nextest` unless specified explicitly, and even then `criterion` harness is incompatible
           # with how `cargo nextest` runs tests. Thus, we run criterion-based benchmark tests manually.
-          ci_run zk f cargo test --release -p vm-benchmark --bench oneshot --bench batch
+          ci_run cargo test --release -p vm-benchmark --bench oneshot --bench batch
 
   loadtest:
     runs-on: [ matterlabs-ci-runner-high-performance ]

zk_toolbox/crates/zk_supervisor/src/commands/contracts.rs

@@ -1,16 +1,16 @@
 use std::path::PathBuf;
 
-use clap::{Parser, ValueEnum};
+use clap::Parser;
 use common::{cmd::Cmd, logger, spinner::Spinner};
 use config::EcosystemConfig;
-use strum::EnumIter;
 use xshell::{cmd, Shell};
 
 use crate::messages::{
     MSG_BUILDING_CONTRACTS, MSG_BUILDING_CONTRACTS_SUCCESS, MSG_BUILDING_L1_CONTRACTS_SPINNER,
     MSG_BUILDING_L2_CONTRACTS_SPINNER, MSG_BUILDING_SYSTEM_CONTRACTS_SPINNER,
-    MSG_BUILD_L1_CONTRACTS_HELP, MSG_BUILD_L2_CONTRACTS_HELP, MSG_BUILD_SYSTEM_CONTRACTS_HELP,
-    MSG_CONTRACTS_DEPS_SPINNER, MSG_NOTHING_TO_BUILD_MSG,
+    MSG_BUILDING_TEST_CONTRACTS_SPINNER, MSG_BUILD_L1_CONTRACTS_HELP, MSG_BUILD_L2_CONTRACTS_HELP,
+    MSG_BUILD_SYSTEM_CONTRACTS_HELP, MSG_BUILD_TEST_CONTRACTS_HELP, MSG_CONTRACTS_DEPS_SPINNER,
+    MSG_NOTHING_TO_BUILD_MSG,
 };
 
 #[derive(Debug, Parser)]
@@ -21,18 +21,22 @@ pub struct ContractsArgs {
     pub l2_contracts: Option<bool>,
     #[clap(long, alias = "sc", help = MSG_BUILD_SYSTEM_CONTRACTS_HELP, default_missing_value = "true", num_args = 0..=1)]
     pub system_contracts: Option<bool>,
+    #[clap(long, alias = "test", help = MSG_BUILD_TEST_CONTRACTS_HELP, default_missing_value = "true", num_args = 0..=1)]
+    pub test_contracts: Option<bool>,
 }
 
 impl ContractsArgs {
     fn contracts(&self) -> Vec<ContractType> {
         if self.l1_contracts.is_none()
             && self.l2_contracts.is_none()
             && self.system_contracts.is_none()
+            && self.test_contracts.is_none()
         {
             return vec![
                 ContractType::L1,
                 ContractType::L2,
                 ContractType::SystemContracts,
+                ContractType::TestContracts,
             ];
         }
 
@@ -47,17 +51,20 @@ impl ContractsArgs {
         if self.system_contracts.unwrap_or(false) {
             contracts.push(ContractType::SystemContracts);
         }
+        if self.test_contracts.unwrap_or(false) {
+            contracts.push(ContractType::TestContracts);
+        }
 
         contracts
     }
 }
 
-#[derive(Debug, ValueEnum, EnumIter, strum::Display, PartialEq, Eq, Clone, Copy)]
-#[strum(serialize_all = "lowercase")]
+#[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum ContractType {
     L1,
     L2,
     SystemContracts,
+    TestContracts,
 }
 
 #[derive(Debug)]
@@ -85,6 +92,11 @@ impl ContractBuilder {
                 cmd: "yarn sc build".to_string(),
                 msg: MSG_BUILDING_SYSTEM_CONTRACTS_SPINNER.to_string(),
             },
+            ContractType::TestContracts => Self {
+                dir: ecosystem.link_to_code.join("etc/contracts-test-data"),
+                cmd: "yarn build".to_string(),
+                msg: MSG_BUILDING_TEST_CONTRACTS_SPINNER.to_string(),
+            },
         }
     }
 

zk_toolbox/crates/zk_supervisor/src/commands/test/rust.rs

@@ -1,12 +1,16 @@
+use std::{path::Path, str::FromStr};
+
 use anyhow::Context;
-use common::{cmd::Cmd, config::global_config, db::wait_for_db, logger};
+use common::{cmd::Cmd, db::wait_for_db, logger};
 use config::EcosystemConfig;
+use url::Url;
 use xshell::{cmd, Shell};
 
 use super::args::rust::RustArgs;
 use crate::{
     commands::database,
-    dals::get_test_dals,
+    dals::{Dal, CORE_DAL_PATH, PROVER_DAL_PATH},
+    defaults::{TEST_DATABASE_PROVER_URL, TEST_DATABASE_SERVER_URL},
     messages::{
         MSG_CARGO_NEXTEST_MISSING_ERR, MSG_CHAIN_NOT_FOUND_ERR, MSG_POSTGRES_CONFIG_NOT_FOUND_ERR,
         MSG_RESETTING_TEST_DATABASES, MSG_UNIT_TESTS_RUN_SUCCESS, MSG_USING_CARGO_NEXTEST,
@@ -17,16 +21,45 @@ pub async fn run(shell: &Shell, args: RustArgs) -> anyhow::Result<()> {
     let ecosystem = EcosystemConfig::from_file(shell)?;
     let chain = ecosystem
         .clone()
-        .load_chain(global_config().chain_name.clone())
+        .load_chain(Some(ecosystem.default_chain))
         .context(MSG_CHAIN_NOT_FOUND_ERR)?;
-    let general_config = chain.get_general_config()?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?;
+    let general_config = chain.get_general_config();
+    let link_to_code = ecosystem.link_to_code;
+
+    let (test_server_url, test_prover_url) = if let Ok(general_config) = general_config {
+        let postgres = general_config
+            .postgres_config
+            .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?;
+
+        (
+            postgres
+                .test_server_url
+                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
+            postgres
+                .test_prover_url
+                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
+        )
+    } else {
+        (
+            TEST_DATABASE_SERVER_URL.to_string(),
+            TEST_DATABASE_PROVER_URL.to_string(),
+        )
+    };
+
+    let dals = vec![
+        Dal {
+            url: Url::from_str(&test_server_url.clone())?,
+            path: CORE_DAL_PATH.to_string(),
+        },
+        Dal {
+            url: Url::from_str(&test_prover_url.clone())?,
+            path: PROVER_DAL_PATH.to_string(),
+        },
+    ];
 
-    reset_test_databases(shell).await?;
+    reset_test_databases(shell, &link_to_code, dals).await?;
 
-    let _dir_guard = shell.push_dir(&ecosystem.link_to_code);
+    let _dir_guard = shell.push_dir(&link_to_code);
 
     let cmd = if nextest_is_installed(shell)? {
         logger::info(MSG_USING_CARGO_NEXTEST);
@@ -43,18 +76,8 @@ pub async fn run(shell: &Shell, args: RustArgs) -> anyhow::Result<()> {
     };
 
     let cmd = cmd
-        .env(
-            "TEST_DATABASE_URL",
-            postgres
-                .test_server_url
-                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
-        )
-        .env(
-            "TEST_PROVER_DATABASE_URL",
-            postgres
-                .test_prover_url
-                .context(MSG_POSTGRES_CONFIG_NOT_FOUND_ERR)?,
-        );
+        .env("TEST_DATABASE_URL", test_server_url)
+        .env("TEST_PROVER_DATABASE_URL", test_prover_url);
     cmd.run()?;
 
     logger::outro(MSG_UNIT_TESTS_RUN_SUCCESS);
@@ -70,9 +93,12 @@ fn nextest_is_installed(shell: &Shell) -> anyhow::Result<bool> {
     Ok(out.contains("cargo-nextest"))
 }
 
-async fn reset_test_databases(shell: &Shell) -> anyhow::Result<()> {
+async fn reset_test_databases(
+    shell: &Shell,
+    link_to_code: &Path,
+    dals: Vec<Dal>,
+) -> anyhow::Result<()> {
     logger::info(MSG_RESETTING_TEST_DATABASES);
-    let ecosystem = EcosystemConfig::from_file(shell)?;
 
     Cmd::new(cmd!(
         shell,
@@ -85,11 +111,11 @@ async fn reset_test_databases(shell: &Shell) -> anyhow::Result<()> {
     ))
     .run()?;
 
-    for dal in get_test_dals(shell)? {
+    for dal in dals {
         let mut url = dal.url.clone();
         url.set_path("");
         wait_for_db(&url, 3).await?;
-        database::reset::reset_database(shell, ecosystem.link_to_code.clone(), dal.clone()).await?;
+        database::reset::reset_database(shell, link_to_code, dal.clone()).await?;
     }
 
     Ok(())

zk_toolbox/crates/zk_supervisor/src/dals.rs

@@ -1,5 +1,3 @@
-use std::str::FromStr;
-
 use anyhow::{anyhow, Context};
 use common::config::global_config;
 use config::{EcosystemConfig, SecretsConfig};
@@ -11,8 +9,8 @@ use crate::{
     messages::{MSG_CHAIN_NOT_FOUND_ERR, MSG_DATABASE_MUST_BE_PRESENTED},
 };
 
-const CORE_DAL_PATH: &str = "core/lib/dal";
-const PROVER_DAL_PATH: &str = "prover/crates/lib/prover_dal";
+pub const CORE_DAL_PATH: &str = "core/lib/dal";
+pub const PROVER_DAL_PATH: &str = "prover/crates/lib/prover_dal";
 
 #[derive(Debug, Clone)]
 pub struct SelectedDals {
@@ -50,10 +48,6 @@ pub fn get_dals(
     Ok(dals)
 }
 
-pub fn get_test_dals(shell: &Shell) -> anyhow::Result<Vec<Dal>> {
-    Ok(vec![get_test_prover_dal(shell)?, get_test_core_dal(shell)?])
-}
-
 pub fn get_prover_dal(shell: &Shell, url: Option<String>) -> anyhow::Result<Dal> {
     let url = if let Some(url) = url {
         Url::parse(&url)?
@@ -94,51 +88,6 @@ pub fn get_core_dal(shell: &Shell, url: Option<String>) -> anyhow::Result<Dal> {
     })
 }
 
-pub fn get_test_core_dal(shell: &Shell) -> anyhow::Result<Dal> {
-    let general_config = get_general_config(shell)?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_DATABASE_MUST_BE_PRESENTED)?;
-
-    let url = Url::from_str(
-        &postgres
-            .test_server_url
-            .clone()
-            .context(MSG_DATABASE_MUST_BE_PRESENTED)?,
-    )?;
-    Ok(Dal {
-        path: CORE_DAL_PATH.to_string(),
-        url,
-    })
-}
-
-pub fn get_test_prover_dal(shell: &Shell) -> anyhow::Result<Dal> {
-    let general_config = get_general_config(shell)?;
-    let postgres = general_config
-        .postgres_config
-        .context(MSG_DATABASE_MUST_BE_PRESENTED)?;
-
-    let url = Url::from_str(
-        &postgres
-            .test_prover_url
-            .clone()
-            .context(MSG_DATABASE_MUST_BE_PRESENTED)?,
-    )?;
-
-    Ok(Dal {
-        path: PROVER_DAL_PATH.to_string(),
-        url,
-    })
-}
-
-fn get_general_config(shell: &Shell) -> anyhow::Result<config::GeneralConfig> {
-    let ecosystem_config = EcosystemConfig::from_file(shell)?;
-    let chain_config = ecosystem_config
-        .load_chain(global_config().chain_name.clone())
-        .context(MSG_CHAIN_NOT_FOUND_ERR)?;
-    chain_config.get_general_config()
-}
-
 fn get_secrets(shell: &Shell) -> anyhow::Result<SecretsConfig> {
     let ecosystem_config = EcosystemConfig::from_file(shell)?;
     let chain_config = ecosystem_config

zk_toolbox/crates/zk_supervisor/src/defaults.rs

@@ -0,0 +1,4 @@
+pub const TEST_DATABASE_SERVER_URL: &str =
+    "postgres://postgres:notsecurepassword@localhost:5433/zksync_local_test";
+pub const TEST_DATABASE_PROVER_URL: &str =
+    "postgres://postgres:notsecurepassword@localhost:5433/prover_local_test";

zk_toolbox/crates/zk_supervisor/src/main.rs

@@ -21,6 +21,7 @@ use crate::commands::{clean::CleanCommands, fmt::FmtArgs};
 
 mod commands;
 mod dals;
+mod defaults;
 mod messages;
 
 #[derive(Parser, Debug)]

zk_toolbox/crates/zk_supervisor/src/messages.rs

@@ -112,10 +112,12 @@ pub(super) const MSG_CONTRACTS_DEPS_SPINNER: &str = "Installing dependencies..";
 pub(super) const MSG_BUILDING_L2_CONTRACTS_SPINNER: &str = "Building L2 contracts..";
 pub(super) const MSG_BUILDING_L1_CONTRACTS_SPINNER: &str = "Building L1 contracts..";
 pub(super) const MSG_BUILDING_SYSTEM_CONTRACTS_SPINNER: &str = "Building system contracts..";
+pub(super) const MSG_BUILDING_TEST_CONTRACTS_SPINNER: &str = "Building test contracts..";
 pub(super) const MSG_BUILDING_CONTRACTS_SUCCESS: &str = "Contracts built successfully";
 pub(super) const MSG_BUILD_L1_CONTRACTS_HELP: &str = "Build L1 contracts";
 pub(super) const MSG_BUILD_L2_CONTRACTS_HELP: &str = "Build L2 contracts";
 pub(super) const MSG_BUILD_SYSTEM_CONTRACTS_HELP: &str = "Build system contracts";
+pub(super) const MSG_BUILD_TEST_CONTRACTS_HELP: &str = "Build test contracts";
 
 // Integration tests related messages
 pub(super) fn msg_integration_tests_run(external_node: bool) -> String {