From 4ab492201a1654a254c0b14a382a2cb67e3cb9e5 Mon Sep 17 00:00:00 2001 From: Agustin Aon <21188659+aon@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:48:43 -0300 Subject: [PATCH] feat(toolbox): add zk_toolbox ci (#1985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ - Adds zk_toolbox CI with ecosystem initialization - Add zk_supervisor integration-tests command - Make paths in ecosystem config absolute ## Why ❔ - Improve CI flow ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- .github/workflows/ci-zk-toolbox-reusable.yml | 100 ++++++++++++++++++ .github/workflows/ci.yml | 11 ++ bin/zk | 2 +- .../tests/revert-and-restart-en.test.ts | 2 +- .../tests/revert-and-restart.test.ts | 2 +- core/tests/ts-integration/src/env.ts | 8 +- core/tests/ts-integration/tests/fees.test.ts | 6 +- core/tests/upgrade-test/tests/upgrade.test.ts | 2 +- etc/utils/.gitignore | 1 + etc/utils/package.json | 13 +++ .../zk/src/utils.ts => etc/utils/src/index.ts | 0 etc/utils/tsconfig.json | 15 +++ .../protocol-upgrade/src/crypto/deployer.ts | 2 +- .../src/hyperchain-upgrade.ts | 2 +- .../src/l1upgrade/deployer.ts | 2 +- .../protocol-upgrade/src/l1upgrade/facets.ts | 2 +- .../src/l2upgrade/deployer.ts | 2 +- .../protocol-upgrade/src/transaction.ts | 2 +- infrastructure/zk/src/clean.ts | 2 +- infrastructure/zk/src/compiler.ts | 2 +- infrastructure/zk/src/config.ts | 2 +- infrastructure/zk/src/contract.ts | 2 +- infrastructure/zk/src/contract_verifier.ts | 2 +- infrastructure/zk/src/database.ts | 2 +- infrastructure/zk/src/docker.ts | 2 +- infrastructure/zk/src/down.ts | 2 +- infrastructure/zk/src/env.ts | 2 +- infrastructure/zk/src/fmt.ts | 2 +- infrastructure/zk/src/format_sql.ts | 2 +- infrastructure/zk/src/hyperchain_wizard.ts | 9 +- infrastructure/zk/src/init.ts | 31 ++++-- infrastructure/zk/src/lint.ts | 2 +- infrastructure/zk/src/prover_setup.ts | 2 +- infrastructure/zk/src/reinit.ts | 7 +- infrastructure/zk/src/run.ts | 2 +- infrastructure/zk/src/server.ts | 2 +- infrastructure/zk/src/setup_en.ts | 2 +- infrastructure/zk/src/spellcheck.ts | 2 +- infrastructure/zk/src/test/integration.ts | 2 +- infrastructure/zk/src/test/test.ts | 2 +- infrastructure/zk/src/up.ts | 10 +- package.json | 2 + zk_toolbox/Cargo.lock | 19 ++++ zk_toolbox/Cargo.toml | 1 + zk_toolbox/crates/config/Cargo.toml | 1 + zk_toolbox/crates/config/src/ecosystem.rs | 19 +++- .../forge_interface/deploy_ecosystem/input.rs | 7 ++ .../zk_inception/src/commands/server.rs | 16 ++- .../crates/zk_inception/src/messages.rs | 1 + .../src/commands/integration_tests.rs | 56 ++++++++++ .../crates/zk_supervisor/src/commands/mod.rs | 1 + zk_toolbox/crates/zk_supervisor/src/main.rs | 11 +- .../crates/zk_supervisor/src/messages.rs | 11 ++ 53 files changed, 358 insertions(+), 56 deletions(-) create mode 100644 .github/workflows/ci-zk-toolbox-reusable.yml create mode 100644 etc/utils/.gitignore create mode 100644 etc/utils/package.json rename infrastructure/zk/src/utils.ts => etc/utils/src/index.ts (100%) create mode 100644 etc/utils/tsconfig.json create mode 100644 zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml new file mode 100644 index 000000000000..c3ef46453f13 --- /dev/null +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -0,0 +1,100 @@ +name: Workflow template for CI jobs for Core Components +on: + workflow_call: + +env: + CLICOLOR: 1 + +jobs: + lint: + name: lint + uses: ./.github/workflows/ci-core-lint-reusable.yml + + build: + runs-on: [matterlabs-ci-runner] + + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + with: + submodules: "recursive" + fetch-depth: 0 + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Start services + run: | + ci_localnet_up + + - name: Build + run: | + ci_run bash -c "cd zk_toolbox && cargo build --release" + + # Compress with tar to avoid permission loss + # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss + - name: Tar zk_toolbox binaries + run: | + tar -C ./zk_toolbox/target/release -cvf zk_toolbox.tar zk_inception zk_supervisor + + - name: Upload zk_toolbox binaries + uses: actions/upload-artifact@v4 + with: + name: zk_toolbox + path: zk_toolbox.tar + compression-level: 0 + + integration_test: + runs-on: [matterlabs-ci-runner] + needs: [build] + + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + with: + submodules: "recursive" + fetch-depth: 0 + + - name: Download zk_toolbox binaries + uses: actions/download-artifact@v4 + with: + name: zk_toolbox + path: . + + - name: Extract zk_toolbox binaries + run: | + tar -xvf zk_toolbox.tar -C ./bin + + - name: Setup environment + run: | + echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV + echo $(pwd)/bin >> $GITHUB_PATH + echo IN_DOCKER=1 >> .env + + - name: Start services + run: | + ci_localnet_up + + - name: Initialize ecosystem + run: | + ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \ + --deploy-ecosystem --l1-rpc-url=http://reth:8545 \ + --server-db-url=postgres://postgres:notsecurepassword@postgres:5432 \ + --server-db-name=zksync_server_localhost_era \ + --prover-db-url=postgres://postgres:notsecurepassword@postgres:5432 \ + --prover-db-name=zksync_prover_localhost_era \ + --ignore-prerequisites --verbose + + - name: Run server + run: | + ci_run zk_inception server --ignore-prerequisites &>server.log & + ci_run sleep 5 + + - name: Run integration tests + run: | + ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose + + - name: Show server.log logs + if: always() + run: ci_run cat server.log || true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21e3104a5dcd..881af2367d31 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,7 @@ jobs: outputs: core: ${{ steps.changed-files.outputs.core_any_changed }} prover: ${{ steps.changed-files.outputs.prover_any_changed }} + zk_toolbox: ${{ steps.changed-files.outputs.zk_toolbox_any_changed }} docs: ${{ steps.changed-files.outputs.docs_any_changed }} all: ${{ steps.changed-files.outputs.all_any_changed }} steps: @@ -60,6 +61,10 @@ jobs: - '!**/*.md' - '!**/*.MD' - 'docker-compose.yml' + zk_toolbox: + - 'zk_toolbox/**' + - '!**/*.md' + - '!**/*.MD' docs: - '**/*.md' - '**/*.MD' @@ -91,6 +96,12 @@ jobs: name: CI for Prover Components uses: ./.github/workflows/ci-prover-reusable.yml + ci-for-zk-toolbox: + needs: changed_files + if: ${{ (needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.zk_toolbox == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }} + name: CI for zk_toolbox + uses: ./.github/workflows/ci-zk-toolbox-reusable.yml + ci-for-docs: needs: changed_files if: needs.changed_files.outputs.docs == 'true' diff --git a/bin/zk b/bin/zk index fec96763b788..868c4e338cdf 100755 --- a/bin/zk +++ b/bin/zk @@ -41,7 +41,7 @@ check_subdirectory check_yarn_version if [ -z "$1" ]; then cd $ZKSYNC_HOME - run_retried yarn install --frozen-lockfile && yarn zk build + run_retried yarn install --frozen-lockfile && yarn utils build && yarn zk build else # can't start this with yarn since it has quirks with `--` as an argument node -- $ZKSYNC_HOME/infrastructure/zk/build/index.js "$@" diff --git a/core/tests/revert-test/tests/revert-and-restart-en.test.ts b/core/tests/revert-test/tests/revert-and-restart-en.test.ts index 7e5931ac8ad3..6edf40a8d2d4 100644 --- a/core/tests/revert-test/tests/revert-and-restart-en.test.ts +++ b/core/tests/revert-test/tests/revert-and-restart-en.test.ts @@ -3,7 +3,7 @@ // NOTE: // main_contract.getTotalBatchesCommitted actually checks the number of batches committed. // main_contract.getTotalBatchesExecuted actually checks the number of batches executed. -import * as utils from 'zk/build/utils'; +import * as utils from 'utils'; import { Tester } from './tester'; import * as zksync from 'zksync-ethers'; import { BigNumber, ethers } from 'ethers'; diff --git a/core/tests/revert-test/tests/revert-and-restart.test.ts b/core/tests/revert-test/tests/revert-and-restart.test.ts index 6381f696283b..92869ab45c8c 100644 --- a/core/tests/revert-test/tests/revert-and-restart.test.ts +++ b/core/tests/revert-test/tests/revert-and-restart.test.ts @@ -1,4 +1,4 @@ -import * as utils from 'zk/build/utils'; +import * as utils from 'utils'; import { Tester } from './tester'; import * as zksync from 'zksync-ethers'; import { BigNumber, Contract, ethers } from 'ethers'; diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index ddbb8227dc60..c440e6b08ea6 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -61,6 +61,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise { diff --git a/infrastructure/zk/src/fmt.ts b/infrastructure/zk/src/fmt.ts index 97be5c571d6f..e58cdbc8e547 100644 --- a/infrastructure/zk/src/fmt.ts +++ b/infrastructure/zk/src/fmt.ts @@ -1,6 +1,6 @@ import { Command } from 'commander'; import { formatSqlxQueries } from './format_sql'; -import * as utils from './utils'; +import * as utils from 'utils'; const EXTENSIONS = ['ts', 'md', 'js']; const CONFIG_PATH = 'etc/prettier-config'; diff --git a/infrastructure/zk/src/format_sql.ts b/infrastructure/zk/src/format_sql.ts index ba1bf263e4cc..7f18d4a46388 100644 --- a/infrastructure/zk/src/format_sql.ts +++ b/infrastructure/zk/src/format_sql.ts @@ -1,5 +1,5 @@ import * as fs from 'fs'; -import * as utils from './utils'; +import * as utils from 'utils'; import { format } from 'sql-formatter'; function formatQuery(query: string) { diff --git a/infrastructure/zk/src/hyperchain_wizard.ts b/infrastructure/zk/src/hyperchain_wizard.ts index 04e9db2a4147..ba4c85454563 100644 --- a/infrastructure/zk/src/hyperchain_wizard.ts +++ b/infrastructure/zk/src/hyperchain_wizard.ts @@ -13,7 +13,7 @@ import fetch from 'node-fetch'; import { up } from './up'; import * as Handlebars from 'handlebars'; import { ProverType, setupProver } from './prover_setup'; -import { announced } from './utils'; +import { announced } from 'utils'; import { DeploymentMode } from './contract'; const title = chalk.blueBright; @@ -49,7 +49,12 @@ export interface BasePromptOptions { async function initHyperchain(envName: string, runObservability: boolean, validiumMode: boolean) { await announced('Initializing hyperchain creation', setupConfiguration(envName, runObservability)); let deploymentMode = validiumMode !== undefined ? DeploymentMode.Validium : DeploymentMode.Rollup; - await init.initHyperCmdAction({ skipSetupCompletely: false, bumpChainId: true, runObservability, deploymentMode }); + await init.initHyperCmdAction({ + skipSetupCompletely: false, + bumpChainId: true, + runObservability, + deploymentMode + }); // TODO: EVM:577 fix hyperchain wizard env.mergeInitToEnv(); diff --git a/infrastructure/zk/src/init.ts b/infrastructure/zk/src/init.ts index d6e30e415e6f..9ed6e178e51e 100644 --- a/infrastructure/zk/src/init.ts +++ b/infrastructure/zk/src/init.ts @@ -1,7 +1,7 @@ import { Command } from 'commander'; -import * as utils from './utils'; -import { announced } from './utils'; +import * as utils from 'utils'; +import { announced } from 'utils'; import { clean } from './clean'; import * as compiler from './compiler'; @@ -161,7 +161,12 @@ export const initDevCmdAction = async ({ await makeEraChainIdSameAsCurrent(); } let deploymentMode = validiumMode !== undefined ? contract.DeploymentMode.Validium : contract.DeploymentMode.Rollup; - await initSetup({ skipEnvSetup, skipSubmodulesCheckout, runObservability, deploymentMode }); + await initSetup({ + skipEnvSetup, + skipSubmodulesCheckout, + runObservability, + deploymentMode + }); if (!skipVerifier) { await deployVerifier(); } @@ -170,7 +175,12 @@ export const initDevCmdAction = async ({ } await initBridgehubStateTransition(); await initDatabase(); - await initHyperchain({ includePaymaster: true, baseTokenName, localLegacyBridgeTesting, deploymentMode }); + await initHyperchain({ + includePaymaster: true, + baseTokenName, + localLegacyBridgeTesting, + deploymentMode + }); if (localLegacyBridgeTesting) { await makeEraAddressSameAsCurrent(); } @@ -214,10 +224,19 @@ export const initHyperCmdAction = async ({ config.bumpChainId(); } if (!skipSetupCompletely) { - await initSetup({ skipEnvSetup: false, skipSubmodulesCheckout: false, runObservability, deploymentMode }); + await initSetup({ + skipEnvSetup: false, + skipSubmodulesCheckout: false, + runObservability, + deploymentMode + }); } await initDatabase(); - await initHyperchain({ includePaymaster: true, baseTokenName, deploymentMode }); + await initHyperchain({ + includePaymaster: true, + baseTokenName, + deploymentMode + }); }; // ########################### Command Definitions ########################### diff --git a/infrastructure/zk/src/lint.ts b/infrastructure/zk/src/lint.ts index fcba41110fb4..84c2c4535c59 100644 --- a/infrastructure/zk/src/lint.ts +++ b/infrastructure/zk/src/lint.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; // Note that `rust` is not noted here, as clippy isn't run via `yarn`. // `rust` option is still supported though. diff --git a/infrastructure/zk/src/prover_setup.ts b/infrastructure/zk/src/prover_setup.ts index 361ae44b8fa0..5a17c9683742 100644 --- a/infrastructure/zk/src/prover_setup.ts +++ b/infrastructure/zk/src/prover_setup.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; import fs from 'fs'; import enquirer from 'enquirer'; import { BasePromptOptions } from './hyperchain_wizard'; diff --git a/infrastructure/zk/src/reinit.ts b/infrastructure/zk/src/reinit.ts index 8535af8e05a8..65f0b73d6540 100644 --- a/infrastructure/zk/src/reinit.ts +++ b/infrastructure/zk/src/reinit.ts @@ -1,7 +1,7 @@ import { Command } from 'commander'; import { up } from './up'; -import { announced } from './utils'; +import { announced } from 'utils'; import { initDevCmdAction, initHyperCmdAction } from './init'; import { DeploymentMode } from './contract'; @@ -20,7 +20,10 @@ const reinitDevCmdAction = async (): Promise => { }); }; -type ReinitHyperCmdActionOptions = { baseTokenName?: string; validiumMode: boolean }; +type ReinitHyperCmdActionOptions = { + baseTokenName?: string; + validiumMode: boolean; +}; const reinitHyperCmdAction = async ({ baseTokenName, validiumMode }: ReinitHyperCmdActionOptions): Promise => { // skipSetupCompletely, because we only want to compile // bumpChainId, because we want to reinitialize hyperchain with a new chain id diff --git a/infrastructure/zk/src/run.ts b/infrastructure/zk/src/run.ts index f0c4994756c1..02e3a15e3c46 100644 --- a/infrastructure/zk/src/run.ts +++ b/infrastructure/zk/src/run.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; import { Wallet } from 'ethers'; import fs from 'fs'; import * as path from 'path'; diff --git a/infrastructure/zk/src/server.ts b/infrastructure/zk/src/server.ts index 923097f5c604..872aff2eb5c3 100644 --- a/infrastructure/zk/src/server.ts +++ b/infrastructure/zk/src/server.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; import { clean } from './clean'; import fs from 'fs'; import * as path from 'path'; diff --git a/infrastructure/zk/src/setup_en.ts b/infrastructure/zk/src/setup_en.ts index 81185ad0cc6b..3d92b326251f 100644 --- a/infrastructure/zk/src/setup_en.ts +++ b/infrastructure/zk/src/setup_en.ts @@ -6,7 +6,7 @@ import fs from 'fs'; import path from 'path'; import { set as setEnv } from './env'; import { setup as setupDb } from './database'; -import * as utils from './utils'; +import * as utils from 'utils'; enum Environment { Mainnet = 'mainnet', diff --git a/infrastructure/zk/src/spellcheck.ts b/infrastructure/zk/src/spellcheck.ts index 4f6553e2c654..8bf78869788d 100644 --- a/infrastructure/zk/src/spellcheck.ts +++ b/infrastructure/zk/src/spellcheck.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; export async function runSpellCheck(pattern: string, useCargo: boolean, useCSpell: boolean) { // Default commands for cSpell and cargo spellcheck diff --git a/infrastructure/zk/src/test/integration.ts b/infrastructure/zk/src/test/integration.ts index 08582a553c79..386ffbef6304 100644 --- a/infrastructure/zk/src/test/integration.ts +++ b/infrastructure/zk/src/test/integration.ts @@ -1,5 +1,5 @@ import { Command } from 'commander'; -import * as utils from '../utils'; +import * as utils from 'utils'; import * as config from '../config'; import deepExtend from 'deep-extend'; diff --git a/infrastructure/zk/src/test/test.ts b/infrastructure/zk/src/test/test.ts index 2aa6fa971d47..2e3202051917 100644 --- a/infrastructure/zk/src/test/test.ts +++ b/infrastructure/zk/src/test/test.ts @@ -1,6 +1,6 @@ import chalk from 'chalk'; import { Command } from 'commander'; -import * as utils from '../utils'; +import * as utils from 'utils'; import * as integration from './integration'; import * as db from '../database'; diff --git a/infrastructure/zk/src/up.ts b/infrastructure/zk/src/up.ts index 2e917b3bea26..6f49dd7d05e1 100644 --- a/infrastructure/zk/src/up.ts +++ b/infrastructure/zk/src/up.ts @@ -1,11 +1,15 @@ import { Command } from 'commander'; -import * as utils from './utils'; +import * as utils from 'utils'; import fs from 'fs'; // Make sure that the volumes exists before starting the containers. export function createVolumes() { - fs.mkdirSync(`${process.env.ZKSYNC_HOME}/volumes/reth/data`, { recursive: true }); - fs.mkdirSync(`${process.env.ZKSYNC_HOME}/volumes/postgres`, { recursive: true }); + fs.mkdirSync(`${process.env.ZKSYNC_HOME}/volumes/reth/data`, { + recursive: true + }); + fs.mkdirSync(`${process.env.ZKSYNC_HOME}/volumes/postgres`, { + recursive: true + }); } export async function up(runObservability: boolean, composeFile?: string) { diff --git a/package.json b/package.json index cdbc8acee00e..b15675264d3e 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "contracts/system-contracts", "etc/contracts-test-data", "etc/ERC20", + "etc/utils", "infrastructure/zk", "infrastructure/local-setup-preparation", "core/tests/revert-test", @@ -32,6 +33,7 @@ "upgrade-test": "yarn workspace upgrade-test", "recovery-test": "yarn workspace recovery-test", "ts-integration": "yarn workspace ts-integration", + "utils": "yarn workspace utils", "zk": "yarn workspace zk" }, "devDependencies": { diff --git a/zk_toolbox/Cargo.lock b/zk_toolbox/Cargo.lock index 927ef514f324..7679313e9d68 100644 --- a/zk_toolbox/Cargo.lock +++ b/zk_toolbox/Cargo.lock @@ -545,6 +545,7 @@ dependencies = [ "clap", "common", "ethers", + "path-absolutize", "rand", "serde", "serde_json", @@ -2301,6 +2302,24 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3145af08024dea9fa9914f381a17b8fc6034dfb00f3a84013f7ff43f29ed4c" +[[package]] +name = "path-absolutize" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4af381fe79fa195b4909485d99f73a80792331df0625188e707854f0b3383f5" +dependencies = [ + "path-dedot", +] + +[[package]] +name = "path-dedot" +version = "3.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ba0ad7e047712414213ff67533e6dd477af0a4e1d14fb52343e53d30ea9397" +dependencies = [ + "once_cell", +] + [[package]] name = "path-slash" version = "0.2.1" diff --git a/zk_toolbox/Cargo.toml b/zk_toolbox/Cargo.toml index ae4b40fa435e..6f9c288438ed 100644 --- a/zk_toolbox/Cargo.toml +++ b/zk_toolbox/Cargo.toml @@ -36,6 +36,7 @@ futures = "0.3.30" human-panic = "2.0" lazy_static = "1.4.0" once_cell = "1.19.0" +path-absolutize = "3.1.1" rand = "0.8.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/zk_toolbox/crates/config/Cargo.toml b/zk_toolbox/crates/config/Cargo.toml index 936cf57498f5..a1fb10760b45 100644 --- a/zk_toolbox/crates/config/Cargo.toml +++ b/zk_toolbox/crates/config/Cargo.toml @@ -15,6 +15,7 @@ anyhow.workspace = true clap.workspace = true common.workspace = true ethers.workspace = true +path-absolutize.workspace = true rand.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/zk_toolbox/crates/config/src/ecosystem.rs b/zk_toolbox/crates/config/src/ecosystem.rs index a76e6a5858a5..1557ab21646f 100644 --- a/zk_toolbox/crates/config/src/ecosystem.rs +++ b/zk_toolbox/crates/config/src/ecosystem.rs @@ -1,5 +1,6 @@ use std::{cell::OnceCell, path::PathBuf}; +use path_absolutize::Absolutize; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use thiserror::Error; use types::{ChainId, L1Network, ProverMode, WalletCreation}; @@ -66,7 +67,11 @@ impl<'de> Deserialize<'de> for EcosystemConfig { Ok(EcosystemConfig { name: config.name.clone(), l1_network: config.l1_network, - link_to_code: config.link_to_code.clone(), + link_to_code: config + .link_to_code + .absolutize() + .expect("Failed to parse zksync-era path") + .to_path_buf(), chains: config.chains.clone(), config: config.config.clone(), default_chain: config.default_chain.clone(), @@ -117,7 +122,11 @@ impl EcosystemConfig { configs: config.configs, l1_batch_commit_data_generator_mode: config.l1_batch_commit_data_generator_mode, l1_network: self.l1_network, - link_to_code: self.link_to_code.clone(), + link_to_code: self + .link_to_code + .absolutize() + .expect("Failed to parse zksync-era path") + .into(), base_token: config.base_token, rocks_db_path: config.rocks_db_path, wallet_creation: config.wallet_creation, @@ -187,7 +196,11 @@ impl EcosystemConfig { EcosystemConfigInternal { name: self.name.clone(), l1_network: self.l1_network, - link_to_code: self.link_to_code.clone(), + link_to_code: self + .link_to_code + .absolutize() + .expect("Failed to parse zksync-era path") + .into(), chains: self.chains.clone(), config: self.config.clone(), default_chain: self.default_chain.clone(), diff --git a/zk_toolbox/crates/config/src/forge_interface/deploy_ecosystem/input.rs b/zk_toolbox/crates/config/src/forge_interface/deploy_ecosystem/input.rs index 585ad407b67e..0998d459ba5c 100644 --- a/zk_toolbox/crates/config/src/forge_interface/deploy_ecosystem/input.rs +++ b/zk_toolbox/crates/config/src/forge_interface/deploy_ecosystem/input.rs @@ -81,6 +81,13 @@ impl Default for Erc20DeploymentConfig { implementation: String::from("TestnetERC20Token.sol"), mint: 10000000000, }, + Erc20DeploymentTokensConfig { + name: String::from("WBTC"), + symbol: String::from("WBTC"), + decimals: 8, + implementation: String::from("TestnetERC20Token.sol"), + mint: 10000000000, + }, Erc20DeploymentTokensConfig { name: String::from("Wrapped Ether"), symbol: String::from("WETH"), diff --git a/zk_toolbox/crates/zk_inception/src/commands/server.rs b/zk_toolbox/crates/zk_inception/src/commands/server.rs index 20ab0f3e32a4..e2d35dd9b792 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/server.rs @@ -1,11 +1,11 @@ use anyhow::Context; -use common::{config::global_config, logger}; +use common::{cmd::Cmd, config::global_config, logger, spinner::Spinner}; use config::{ChainConfig, EcosystemConfig}; -use xshell::Shell; +use xshell::{cmd, Shell}; use crate::{ commands::args::RunServerArgs, - messages::{MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, + messages::{MSG_BUILDING_L1_CONTRACTS, MSG_CHAIN_NOT_INITIALIZED, MSG_STARTING_SERVER}, server::{RunServer, ServerMode}, }; @@ -18,11 +18,21 @@ pub fn run(shell: &Shell, args: RunServerArgs) -> anyhow::Result<()> { .context(MSG_CHAIN_NOT_INITIALIZED)?; logger::info(MSG_STARTING_SERVER); + + build_l1_contracts(shell, &ecosystem_config)?; run_server(args, &chain_config, shell)?; Ok(()) } +fn build_l1_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let _dir_guard = shell.push_dir(ecosystem_config.path_to_foundry()); + let spinner = Spinner::new(MSG_BUILDING_L1_CONTRACTS); + Cmd::new(cmd!(shell, "yarn build")).run()?; + spinner.finish(); + Ok(()) +} + fn run_server( args: RunServerArgs, chain_config: &ChainConfig, diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 799f1a5e2d7a..2e328baa3a58 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -164,6 +164,7 @@ pub(super) const MSG_FAILED_TO_FIND_ECOSYSTEM_ERR: &str = "Failed to find ecosys /// Server related messages pub(super) const MSG_STARTING_SERVER: &str = "Starting server"; pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; +pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts..."; /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs new file mode 100644 index 000000000000..c5b1229dd2ce --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/integration_tests.rs @@ -0,0 +1,56 @@ +use common::{cmd::Cmd, logger, spinner::Spinner}; +use config::EcosystemConfig; +use xshell::{cmd, Shell}; + +use crate::messages::{ + MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS, MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES, + MSG_INTEGRATION_TESTS_RUN_INFO, MSG_INTEGRATION_TESTS_RUN_SUCCESS, +}; + +const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; +const CONTRACTS_TEST_DATA_PATH: &str = "etc/contracts-test-data"; + +pub fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); + + logger::info(MSG_INTEGRATION_TESTS_RUN_INFO); + + build_repository(shell, &ecosystem_config)?; + build_test_contracts(shell, &ecosystem_config)?; + + Cmd::new( + cmd!(shell, "yarn jest --forceExit --testTimeout 60000") + .env("CHAIN_NAME", ecosystem_config.default_chain), + ) + .with_force_run() + .run()?; + + logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS); + + Ok(()) +} + +fn build_repository(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let _dir_guard = shell.push_dir(&ecosystem_config.link_to_code); + let spinner = Spinner::new(MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES); + + Cmd::new(cmd!(shell, "yarn install --frozen-lockfile")).run()?; + Cmd::new(cmd!(shell, "yarn utils build")).run()?; + + spinner.finish(); + Ok(()) +} + +fn build_test_contracts(shell: &Shell, ecosystem_config: &EcosystemConfig) -> anyhow::Result<()> { + let spinner = Spinner::new(MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS); + + Cmd::new(cmd!(shell, "yarn build")).run()?; + Cmd::new(cmd!(shell, "yarn build-yul")).run()?; + + let _dir_guard = shell.push_dir(ecosystem_config.link_to_code.join(CONTRACTS_TEST_DATA_PATH)); + Cmd::new(cmd!(shell, "yarn build")).run()?; + + spinner.finish(); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs index 8fd0a6be869b..98d4cdfe990d 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/mod.rs @@ -1 +1,2 @@ pub mod database; +pub mod integration_tests; diff --git a/zk_toolbox/crates/zk_supervisor/src/main.rs b/zk_toolbox/crates/zk_supervisor/src/main.rs index 24daaba35347..ab5629465a88 100644 --- a/zk_toolbox/crates/zk_supervisor/src/main.rs +++ b/zk_toolbox/crates/zk_supervisor/src/main.rs @@ -6,7 +6,10 @@ use common::{ init_prompt_theme, logger, }; use config::EcosystemConfig; -use messages::msg_global_chain_does_not_exist; +use messages::{ + msg_global_chain_does_not_exist, MSG_SUBCOMMAND_DATABASE_ABOUT, + MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT, +}; use xshell::Shell; mod commands; @@ -24,9 +27,10 @@ struct Supervisor { #[derive(Subcommand, Debug)] enum SupervisorSubcommands { - /// Database related commands - #[command(subcommand)] + #[command(subcommand, about = MSG_SUBCOMMAND_DATABASE_ABOUT)] Database(DatabaseCommands), + #[command(about = MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT)] + IntegrationTests, } #[derive(Parser, Debug)] @@ -89,6 +93,7 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(args: Supervisor, shell: &Shell) -> anyhow::Result<()> { match args.command { SupervisorSubcommands::Database(command) => commands::database::run(shell, command).await?, + SupervisorSubcommands::IntegrationTests => commands::integration_tests::run(shell)?, } Ok(()) } diff --git a/zk_toolbox/crates/zk_supervisor/src/messages.rs b/zk_toolbox/crates/zk_supervisor/src/messages.rs index 97152396b5e5..31bdb0eb9b1d 100644 --- a/zk_toolbox/crates/zk_supervisor/src/messages.rs +++ b/zk_toolbox/crates/zk_supervisor/src/messages.rs @@ -4,6 +4,10 @@ pub(super) fn msg_global_chain_does_not_exist(chain: &str, available_chains: &st format!("Chain with name {chain} doesnt exist, please choose one of: {available_chains}") } +// Subcommands help +pub(super) const MSG_SUBCOMMAND_DATABASE_ABOUT: &str = "Database related commands"; +pub(super) const MSG_SUBCOMMAND_INTEGRATION_TESTS_ABOUT: &str = "Run integration tests"; + // Database related messages pub(super) const MSG_NO_DATABASES_SELECTED: &str = "No databases selected"; pub(super) fn msg_database_info(gerund_verb: &str) -> String { @@ -57,3 +61,10 @@ pub(super) fn msg_database_new_migration_loading(dal: &str) -> String { format!("Creating new database migration for dal {}...", dal) } pub(super) const MSG_DATABASE_NEW_MIGRATION_SUCCESS: &str = "Migration created successfully"; + +// Integration tests related messages +pub(super) const MSG_INTEGRATION_TESTS_RUN_INFO: &str = "Running integration tests"; +pub(super) const MSG_INTEGRATION_TESTS_RUN_SUCCESS: &str = "Integration tests ran successfully"; +pub(super) const MSG_INTEGRATION_TESTS_BUILDING_DEPENDENCIES: &str = + "Building repository dependencies..."; +pub(super) const MSG_INTEGRATION_TESTS_BUILDING_CONTRACTS: &str = "Building test contracts...";