feat(toolbox): add zk_toolbox ci

.github/workflows/ci-zk-toolbox-reusable.yml

@@ -0,0 +1,104 @@
+name: Workflow template for CI jobs for Core Components
+on:
+  workflow_call:
+
+env:
+  CLICOLOR: 1
+
+jobs:
+  lint:
+    name: lint
+    uses: ./.github/workflows/ci-core-lint-reusable.yml
+
+  build:
+    runs-on: [matterlabs-ci-runner]
+
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+        with:
+          submodules: "recursive"
+          fetch-depth: 0
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          ci_localnet_up
+
+      - name: Build
+        run: |
+          ci_run bash -c "cd zk_toolbox && cargo build --release"
+
+      # Compress with tar to avoid permission loss
+      # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+      - name: Tar zk_toolbox binaries
+        run: |
+          tar -C ./zk_toolbox/target/release -cvf zk_toolbox.tar zk_inception zk_supervisor
+
+      - name: Upload zk_toolbox binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: zk_toolbox
+          path: zk_toolbox.tar
+          compression-level: 0
+
+  integration_test:
+    runs-on: [matterlabs-ci-runner]
+    needs: [build]
+
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+        with:
+          submodules: "recursive"
+          fetch-depth: 0
+
+      - name: Download zk_toolbox binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: zk_toolbox
+          path: .
+
+      - name: Extract zk_toolbox binaries
+        run: |
+          tar -xvf zk_toolbox.tar -C ./bin
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          ci_localnet_up
+
+      - name: Setup zk
+        run: |
+          ci_run zk
+
+      - name: Initialize ecosystem
+        run: |
+          ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \
+          --deploy-ecosystem --l1-rpc-url=http://reth:8545 \
+          --server-db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --server-db-name=zksync_server_localhost_era \
+          --prover-db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --prover-db-name=zksync_prover_localhost_era \
+          --ignore-prerequisites --verbose
+
+      - name: Run server
+        run: |
+          ci_run zk_inception server --ignore-prerequisites &>server.log &
+          ci_run sleep 5
+
+      - name: Run integration tests
+        run: |
+          ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose
+
+      - name: Show server.log logs
+        if: always()
+        run: ci_run cat server.log || true

.github/workflows/ci.yml

@@ -20,6 +20,7 @@ jobs:
     outputs:
       core: ${{ steps.changed-files.outputs.core_any_changed }}
       prover: ${{ steps.changed-files.outputs.prover_any_changed }}
+      zk_toolbox: ${{ steps.changed-files.outputs.zk_toolbox_any_changed }}
       docs: ${{ steps.changed-files.outputs.docs_any_changed }}
       all: ${{ steps.changed-files.outputs.all_any_changed }}
     steps:
@@ -60,6 +61,10 @@ jobs:
               - '!**/*.md'
               - '!**/*.MD'
               - 'docker-compose.yml'
+            zk_toolbox:
+              - 'zk_toolbox/**'
+              - '!**/*.md'
+              - '!**/*.MD'
             docs:
               - '**/*.md'
               - '**/*.MD'
@@ -91,6 +96,12 @@ jobs:
     name: CI for Prover Components
     uses: ./.github/workflows/ci-prover-reusable.yml
 
+  ci-for-zk-toolbox:
+    needs: changed_files
+    if: ${{ (needs.changed_files.outputs.zk_toolbox == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }}
+    name: CI for zk_toolbox
+    uses: ./.github/workflows/ci-zk-toolbox-reusable.yml
+
   ci-for-docs:
     needs: changed_files
     if: needs.changed_files.outputs.docs == 'true'

core/tests/ts-integration/src/env.ts

@@ -61,6 +61,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
 
     let generalConfig = loadConfig(pathToHome, chain, 'general.yaml');
     let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml');
+    let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml');
 
     const network = ecosystem.l1_network;
     let mainWalletPK = getMainWalletPk(pathToHome, network);
@@ -71,7 +72,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
     const l2Provider = new zksync.Provider(l2NodeUrl);
     const baseTokenAddress = await l2Provider.getBaseTokenContractAddress();
 
-    const l1NodeUrl = ecosystem.l1_rpc_url;
+    const l1NodeUrl = secretsConfig.l1.l1_rpc_url;
     const wsL2NodeUrl = generalConfig.api.web3_json_rpc.ws_url;
 
     const contractVerificationUrl = generalConfig.contract_verifier.url;
@@ -80,9 +81,12 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
     // wBTC is chosen because it has decimals different from ETH (8 instead of 18).
     // Using this token will help us to detect decimals-related errors.
     // but if it's not available, we'll use the first token from the list.
-    let token = tokens.tokens['wBTC'];
+    let token = tokens.tokens['WBTC'];
     if (token === undefined) {
         token = Object.values(tokens.tokens)[0];
+        if (token.symbol == 'WETH') {
+            token = Object.values(tokens.tokens)[1];
+        }
     }
     const weth = tokens.tokens['WETH'];
     let baseToken;

zk_toolbox/Cargo.toml

@@ -36,6 +36,7 @@ futures = "0.3.30"
 human-panic = "2.0"
 lazy_static = "1.4.0"
 once_cell = "1.19.0"
+path-absolutize = "3.1.1"
 rand = "0.8.5"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

zk_toolbox/crates/config/Cargo.toml

@@ -15,6 +15,7 @@ anyhow.workspace = true
 clap.workspace = true
 common.workspace = true
 ethers.workspace = true
+path-absolutize.workspace = true
 rand.workspace = true
 serde.workspace = true
 serde_json.workspace = true

zk_toolbox/crates/config/src/ecosystem.rs

@@ -1,5 +1,6 @@
 use std::{cell::OnceCell, path::PathBuf};
 
+use path_absolutize::Absolutize;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use thiserror::Error;
 use types::{ChainId, L1Network, ProverMode, WalletCreation};
@@ -66,7 +67,11 @@ impl<'de> Deserialize<'de> for EcosystemConfig {
         Ok(EcosystemConfig {
             name: config.name.clone(),
             l1_network: config.l1_network,
-            link_to_code: config.link_to_code.clone(),
+            link_to_code: config
+                .link_to_code
+                .absolutize()
+                .expect("Failed to parse zksync-era path")
+                .to_path_buf(),
             chains: config.chains.clone(),
             config: config.config.clone(),
             default_chain: config.default_chain.clone(),
@@ -117,7 +122,11 @@ impl EcosystemConfig {
             configs: config.configs,
             l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode,
             l1_network: self.l1_network,
-            link_to_code: self.link_to_code.clone(),
+            link_to_code: self
+                .link_to_code
+                .absolutize()
+                .expect("Failed to parse zksync-era path")
+                .into(),
             base_token: config.base_token,
             rocks_db_path: config.rocks_db_path,
             wallet_creation: config.wallet_creation,
@@ -187,7 +196,11 @@ impl EcosystemConfig {
         EcosystemConfigInternal {
             name: self.name.clone(),
             l1_network: self.l1_network,
-            link_to_code: self.link_to_code.clone(),
+            link_to_code: self
+                .link_to_code
+                .absolutize()
+                .expect("Failed to parse zksync-era path")
+                .into(),
             chains: self.chains.clone(),
             config: self.config.clone(),
             default_chain: self.default_chain.clone(),

zk_toolbox/crates/config/src/forge_interface/deploy_ecosystem/input.rs

@@ -81,6 +81,13 @@ impl Default for Erc20DeploymentConfig {
                     implementation: String::from("TestnetERC20Token.sol"),
                     mint: 10000000000,
                 },
+                Erc20DeploymentTokensConfig {
+                    name: String::from("WBTC"),
+                    symbol: String::from("WBTC"),
+                    decimals: 8,
+                    implementation: String::from("TestnetERC20Token.sol"),
+                    mint: 10000000000,
+                },
                 Erc20DeploymentTokensConfig {
                     name: String::from("Wrapped Ether"),
                     symbol: String::from("WETH"),

zk_toolbox/crates/zk_inception/src/commands/server.rs

@@ -1,11 +1,11 @@
 use anyhow::Context;
-use common::{config::global_config, logger};
+use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner};
 use config::{ChainConfig, EcosystemConfig};
-use xshell::Shell;
+use xshell::{cmd, Shell};
 
 use crate::{
     commands::args::RunServerArgs,
-    messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER},
+    messages::{MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER},
     server::{RunServer, ServerMode},
 };
 
@@ -18,11 +18,21 @@ pub fn run(shell: &Shell, args: RunServerArgs) -> anyhow::Result<()> {
         .context(MSG_CHAIN_NOT_INITIALIZED)?;
 
     logger::info(MSG_STARTING_SERVER);
+
+    build_l1_contracts(shell, &ecosystem_config)?;
     run_server(args, &chain_config, shell)?;
 
     Ok(())
 }
 
+fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> {
+    let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry());
+    let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS);
+    Cmd::new(cmd!(shell, "yarn build")).run()?;
+    spinner.finish();
+    Ok(())
+}
+
 fn run_server(
     args: RunServerArgs,
     chain_config: &ChainConfig,

zk_toolbox/crates/zk_inception/src/messages.rs

@@ -164,6 +164,7 @@ pub(super) const MSG_FAILED_TO_FIND_ECOSYSTEM_ERR: &str = "Failed to find ecosys
 /// Server related messages
 pub(super) const MSG_STARTING_SERVER: &str = "Starting server";
 pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server";
+pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts...";
 
 /// Forge utils related messages
 pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set";

zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs

@@ -0,0 +1,44 @@
+use common::{cmd::Cmd, logger, spinner::Spinner};
+use config::EcosystemConfig;
+use xshell::{cmd, Shell};
+
+use crate::messages::{
+    MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, MSG_INTEGRATION_TESTS_RUN_INFO,
+    MSG_INTEGRATION_TESTS_RUN_SUCCESS,
+};
+
+const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration";
+const CONTRACTS_TEST_DATA_PATH: &str = "etc/contracts-test-data";
+
+pub fn run(shell: &Shell) -> anyhow::Result<()> {
+    let ecosystem_config = EcosystemConfig::from_file(shell)?;
+    shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH));
+
+    logger::info(MSG_INTEGRATION_TESTS_RUN_INFO);
+
+    build_test_contracts(shell, &ecosystem_config)?;
+
+    Cmd::new(
+        cmd!(shell, "yarn jest --forceExit --testTimeout 60000")
+            .env("CHAIN_NAME", ecosystem_config.default_chain),
+    )
+    .with_force_run()
+    .run()?;
+
+    logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS);
+
+    Ok(())
+}
+
+fn build_test_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> {
+    let spinner = Spinner::new(MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS);
+
+    Cmd::new(cmd!(shell, "yarn build")).run()?;
+    Cmd::new(cmd!(shell, "yarn build-yul")).run()?;
+
+    let _dir_guard = shell.push_dir(ecosystem_config.link_to_code.join(CONTRACTS_TEST_DATA_PATH));
+    Cmd::new(cmd!(shell, "yarn build")).run()?;
+
+    spinner.finish();
+    Ok(())
+}

zk_toolbox/crates/zk_supervisor/src/commands/mod.rs

@@ -1 +1,2 @@
 pub mod database;
+pub mod integration_tests;