feat(toolbox): add zk_toolbox ci (#1985)

## What ❔ - Adds zk_toolbox CI with ecosystem initialization - Add zk_supervisor integration-tests command - Make paths in ecosystem config absolute ## Why ❔ - Improve CI flow ## Checklist   - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`.
matter-labs · Jun 5, 2024 · 4ab4922 · 4ab4922
1 parent 6c726eb
commit 4ab4922
Show file tree

Hide file tree

Showing 53 changed files with 358 additions and 56 deletions.
diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml
@@ -0,0 +1,100 @@
+name: Workflow template for CI jobs for Core Components
+on:
+  workflow_call:
+
+env:
+  CLICOLOR: 1
+
+jobs:
+  lint:
+    name: lint
+    uses: ./.github/workflows/ci-core-lint-reusable.yml
+
+  build:
+    runs-on: [matterlabs-ci-runner]
+
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+        with:
+          submodules: "recursive"
+          fetch-depth: 0
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          ci_localnet_up
+
+      - name: Build
+        run: |
+          ci_run bash -c "cd zk_toolbox && cargo build --release"
+
+      # Compress with tar to avoid permission loss
+      # https://github.com/actions/upload-artifact?tab=readme-ov-file#permission-loss
+      - name: Tar zk_toolbox binaries
+        run: |
+          tar -C ./zk_toolbox/target/release -cvf zk_toolbox.tar zk_inception zk_supervisor
+
+      - name: Upload zk_toolbox binaries
+        uses: actions/upload-artifact@v4
+        with:
+          name: zk_toolbox
+          path: zk_toolbox.tar
+          compression-level: 0
+
+  integration_test:
+    runs-on: [matterlabs-ci-runner]
+    needs: [build]
+
+    steps:
+      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+        with:
+          submodules: "recursive"
+          fetch-depth: 0
+
+      - name: Download zk_toolbox binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: zk_toolbox
+          path: .
+
+      - name: Extract zk_toolbox binaries
+        run: |
+          tar -xvf zk_toolbox.tar -C ./bin
+
+      - name: Setup environment
+        run: |
+          echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV
+          echo $(pwd)/bin >> $GITHUB_PATH
+          echo IN_DOCKER=1 >> .env
+
+      - name: Start services
+        run: |
+          ci_localnet_up
+
+      - name: Initialize ecosystem
+        run: |
+          ci_run zk_inception ecosystem init --deploy-paymaster --deploy-erc20 \
+          --deploy-ecosystem --l1-rpc-url=http://reth:8545 \
+          --server-db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --server-db-name=zksync_server_localhost_era \
+          --prover-db-url=postgres://postgres:notsecurepassword@postgres:5432 \
+          --prover-db-name=zksync_prover_localhost_era \
+          --ignore-prerequisites --verbose
+
+      - name: Run server
+        run: |
+          ci_run zk_inception server --ignore-prerequisites &>server.log &
+          ci_run sleep 5
+
+      - name: Run integration tests
+        run: |
+          ci_run zk_supervisor integration-tests --ignore-prerequisites --verbose
+
+      - name: Show server.log logs
+        if: always()
+        run: ci_run cat server.log || true
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -20,6 +20,7 @@ jobs:
     outputs:
       core: ${{ steps.changed-files.outputs.core_any_changed }}
       prover: ${{ steps.changed-files.outputs.prover_any_changed }}
+      zk_toolbox: ${{ steps.changed-files.outputs.zk_toolbox_any_changed }}
       docs: ${{ steps.changed-files.outputs.docs_any_changed }}
       all: ${{ steps.changed-files.outputs.all_any_changed }}
     steps:
@@ -60,6 +61,10 @@ jobs:
               - '!**/*.md'
               - '!**/*.MD'
               - 'docker-compose.yml'
+            zk_toolbox:
+              - 'zk_toolbox/**'
+              - '!**/*.md'
+              - '!**/*.MD'
             docs:
               - '**/*.md'
               - '**/*.MD'
@@ -91,6 +96,12 @@ jobs:
     name: CI for Prover Components
     uses: ./.github/workflows/ci-prover-reusable.yml
 
+  ci-for-zk-toolbox:
+    needs: changed_files
+    if: ${{ (needs.changed_files.outputs.core == 'true' || needs.changed_files.outputs.zk_toolbox == 'true' || needs.changed_files.outputs.all == 'true') && !contains(github.ref_name, 'release-please--branches') }}
+    name: CI for zk_toolbox
+    uses: ./.github/workflows/ci-zk-toolbox-reusable.yml
+
   ci-for-docs:
     needs: changed_files
     if: needs.changed_files.outputs.docs == 'true'

diff --git a/bin/zk b/bin/zk
@@ -41,7 +41,7 @@ check_subdirectory
 check_yarn_version
 if [ -z "$1" ]; then
     cd $ZKSYNC_HOME
-    run_retried yarn install --frozen-lockfile && yarn zk build
+    run_retried yarn install --frozen-lockfile && yarn utils build && yarn zk build
 else
     # can't start this with yarn since it has quirks with `--` as an argument
     node -- $ZKSYNC_HOME/infrastructure/zk/build/index.js "$@"

diff --git a/core/tests/revert-test/tests/revert-and-restart-en.test.ts b/core/tests/revert-test/tests/revert-and-restart-en.test.ts
@@ -3,7 +3,7 @@
 // NOTE:
 // main_contract.getTotalBatchesCommitted actually checks the number of batches committed.
 // main_contract.getTotalBatchesExecuted actually checks the number of batches executed.
-import * as utils from 'zk/build/utils';
+import * as utils from 'utils';
 import { Tester } from './tester';
 import * as zksync from 'zksync-ethers';
 import { BigNumber, ethers } from 'ethers';

diff --git a/core/tests/revert-test/tests/revert-and-restart.test.ts b/core/tests/revert-test/tests/revert-and-restart.test.ts
@@ -1,4 +1,4 @@
-import * as utils from 'zk/build/utils';
+import * as utils from 'utils';
 import { Tester } from './tester';
 import * as zksync from 'zksync-ethers';
 import { BigNumber, Contract, ethers } from 'ethers';

diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts
@@ -61,6 +61,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
 
     let generalConfig = loadConfig(pathToHome, chain, 'general.yaml');
     let genesisConfig = loadConfig(pathToHome, chain, 'genesis.yaml');
+    let secretsConfig = loadConfig(pathToHome, chain, 'secrets.yaml');
 
     const network = ecosystem.l1_network;
     let mainWalletPK = getMainWalletPk(pathToHome, network);
@@ -71,7 +72,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
     const l2Provider = new zksync.Provider(l2NodeUrl);
     const baseTokenAddress = await l2Provider.getBaseTokenContractAddress();
 
-    const l1NodeUrl = ecosystem.l1_rpc_url;
+    const l1NodeUrl = secretsConfig.l1.l1_rpc_url;
     const wsL2NodeUrl = generalConfig.api.web3_json_rpc.ws_url;
 
     const contractVerificationUrl = generalConfig.contract_verifier.url;
@@ -80,9 +81,12 @@ async function loadTestEnvironmentFromFile(chain: string): Promise<TestEnvironme
     // wBTC is chosen because it has decimals different from ETH (8 instead of 18).
     // Using this token will help us to detect decimals-related errors.
     // but if it's not available, we'll use the first token from the list.
-    let token = tokens.tokens['wBTC'];
+    let token = tokens.tokens['WBTC'];
     if (token === undefined) {
         token = Object.values(tokens.tokens)[0];
+        if (token.symbol == 'WETH') {
+            token = Object.values(tokens.tokens)[1];
+        }
     }
     const weth = tokens.tokens['WETH'];
     let baseToken;

diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts
@@ -9,7 +9,7 @@
  * sure that the test is maintained does not get broken.
  *
  */
-import * as utils from 'zk/build/utils';
+import * as utils from 'utils';
 import * as fs from 'fs';
 import { TestMaster } from '../src/index';
 
@@ -258,10 +258,10 @@ async function updateReport(
     const l2EstimatedPriceAsNumber = +ethers.utils.formatEther(estimatedPrice);
 
     const gasReport = `Gas price ${newL1GasPrice / 1000000000} gwei:
-    L1 cost ${expectedL1Price}, 
+    L1 cost ${expectedL1Price},
     L2 estimated cost: ${l2EstimatedPriceAsNumber}
     Estimated Gain: ${expectedL1Price / l2EstimatedPriceAsNumber}
-    L2 cost: ${l2PriceAsNumber}, 
+    L2 cost: ${l2PriceAsNumber},
     Gain: ${expectedL1Price / l2PriceAsNumber}\n`;
     console.log(gasReport);
 

diff --git a/core/tests/upgrade-test/tests/upgrade.test.ts b/core/tests/upgrade-test/tests/upgrade.test.ts
@@ -1,4 +1,4 @@
-import * as utils from 'zk/build/utils';
+import * as utils from 'utils';
 import { Tester } from './tester';
 import * as zksync from 'zksync-ethers';
 import { BigNumber, BigNumberish, ethers } from 'ethers';

diff --git a/etc/utils/.gitignore b/etc/utils/.gitignore
@@ -0,0 +1 @@
+/build
diff --git a/etc/utils/package.json b/etc/utils/package.json
@@ -0,0 +1,13 @@
+{
+    "name": "utils",
+    "version": "0.1.0",
+    "license": "MIT",
+    "main": "build/index.js",
+    "types": "build/index.d.ts",
+    "scripts": {
+        "build": "tsc"
+    },
+    "dependencies": {
+        "chalk": "^4.0.0"
+    }
+}
diff --git a/infrastructure/zk/src/utils.ts → etc/utils/src/index.ts b/infrastructure/zk/src/utils.ts → etc/utils/src/index.ts
diff --git a/etc/utils/tsconfig.json b/etc/utils/tsconfig.json
@@ -0,0 +1,15 @@
+{
+    "compilerOptions": {
+        "target": "es2019",
+        "module": "commonjs",
+        "outDir": "build",
+        "strict": true,
+        "esModuleInterop": true,
+        "noEmitOnError": true,
+        "skipLibCheck": true,
+        "declaration": true
+    },
+    "files": [
+        "src/index.ts"
+    ]
+}
diff --git a/infrastructure/protocol-upgrade/src/crypto/deployer.ts b/infrastructure/protocol-upgrade/src/crypto/deployer.ts
@@ -1,4 +1,4 @@
-import { spawn } from 'zk/build/utils';
+import { spawn } from 'utils';
 
 export async function deployVerifier(
     l1Rpc: string,

diff --git a/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts b/infrastructure/protocol-upgrade/src/hyperchain-upgrade.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import { spawn } from 'zk/build/utils';
+import { spawn } from 'utils';
 import fs from 'fs';
 import { ethers } from 'ethers';
 

diff --git a/infrastructure/protocol-upgrade/src/l1upgrade/deployer.ts b/infrastructure/protocol-upgrade/src/l1upgrade/deployer.ts
@@ -1,4 +1,4 @@
-import { spawn } from 'zk/build/utils';
+import { spawn } from 'utils';
 
 export async function callFacetDeployer(
     l1RpcProvider: string,

diff --git a/infrastructure/protocol-upgrade/src/l1upgrade/facets.ts b/infrastructure/protocol-upgrade/src/l1upgrade/facets.ts
@@ -1,6 +1,6 @@
 import fs from 'fs';
 import { Command } from 'commander';
-import { spawn } from 'zk/build/utils';
+import { spawn } from 'utils';
 import { getFacetCutsFileName, getFacetsFileName, getUpgradePath } from '../utils';
 import { callFacetDeployer } from './deployer';
 

diff --git a/infrastructure/protocol-upgrade/src/l2upgrade/deployer.ts b/infrastructure/protocol-upgrade/src/l2upgrade/deployer.ts
@@ -1,4 +1,4 @@
-import { spawn } from 'zk/build/utils';
+import { spawn } from 'utils';
 
 export async function callSystemContractDeployer(
     l1RpcProvider: string,

diff --git a/infrastructure/protocol-upgrade/src/transaction.ts b/infrastructure/protocol-upgrade/src/transaction.ts
@@ -23,7 +23,7 @@ import {
 } from './utils';
 import fs from 'fs';
 import { Command } from 'commander';
-import { web3Url } from 'zk/build/utils';
+import { web3Url } from 'utils';
 import * as path from 'path';
 
 const testConfigPath = path.join(process.env.ZKSYNC_HOME as string, `etc/test_config/constant`);

diff --git a/infrastructure/zk/src/clean.ts b/infrastructure/zk/src/clean.ts
@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 import * as fs from 'fs';
 import * as path from 'path';
-import { confirmAction } from './utils';
+import { confirmAction } from 'utils';
 import * as down from './down';
 
 export function clean(path: string) {

diff --git a/infrastructure/zk/src/compiler.ts b/infrastructure/zk/src/compiler.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 
 export async function compileTestContracts() {
     await utils.spawn('yarn workspace contracts-test-data build');

diff --git a/infrastructure/zk/src/config.ts b/infrastructure/zk/src/config.ts
@@ -5,7 +5,7 @@ import deepExtend from 'deep-extend';
 import * as env from './env';
 import path from 'path';
 import dotenv from 'dotenv';
-import { unpackStringSemVer } from './utils';
+import { unpackStringSemVer } from 'utils';
 
 function loadConfigFile(configPath: string, stack: string[] = []) {
     if (stack.includes(configPath)) {

diff --git a/infrastructure/zk/src/contract.ts b/infrastructure/zk/src/contract.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 import * as env from './env';
 import fs from 'fs';
 

diff --git a/infrastructure/zk/src/contract_verifier.ts b/infrastructure/zk/src/contract_verifier.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 
 export async function contractVerifier() {
     await utils.spawn(`cargo run --bin zksync_contract_verifier --release`);

diff --git a/infrastructure/zk/src/database.ts b/infrastructure/zk/src/database.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 
 export async function reset(opts: DbOpts) {
     await utils.confirmAction();

diff --git a/infrastructure/zk/src/docker.ts b/infrastructure/zk/src/docker.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 
 const IMAGES = [
     'server-v2',

diff --git a/infrastructure/zk/src/down.ts b/infrastructure/zk/src/down.ts
@@ -1,5 +1,5 @@
 import { Command } from 'commander';
-import * as utils from './utils';
+import * as utils from 'utils';
 import * as fs from 'fs';
 
 export async function down() {

diff --git a/infrastructure/zk/src/env.ts b/infrastructure/zk/src/env.ts
@@ -1,7 +1,7 @@
 import { Command } from 'commander';
 import fs from 'fs';
 import dotenv from 'dotenv';
-import * as utils from './utils';
+import * as utils from 'utils';
 import * as config from './config';
 
 export const getAvailableEnvsFromFiles = () => {

diff --git a/infrastructure/zk/src/fmt.ts b/infrastructure/zk/src/fmt.ts
@@ -1,6 +1,6 @@
 import { Command } from 'commander';
 import { formatSqlxQueries } from './format_sql';
-import * as utils from './utils';
+import * as utils from 'utils';
 
 const EXTENSIONS = ['ts', 'md', 'js'];
 const CONFIG_PATH = 'etc/prettier-config';

diff --git a/infrastructure/zk/src/format_sql.ts b/infrastructure/zk/src/format_sql.ts
@@ -1,5 +1,5 @@
 import * as fs from 'fs';
-import * as utils from './utils';
+import * as utils from 'utils';
 import { format } from 'sql-formatter';
 
 function formatQuery(query: string) {

diff --git a/infrastructure/zk/src/hyperchain_wizard.ts b/infrastructure/zk/src/hyperchain_wizard.ts
@@ -13,7 +13,7 @@ import fetch from 'node-fetch';
 import { up } from './up';
 import * as Handlebars from 'handlebars';
 import { ProverType, setupProver } from './prover_setup';
-import { announced } from './utils';
+import { announced } from 'utils';
 import { DeploymentMode } from './contract';
 
 const title = chalk.blueBright;
@@ -49,7 +49,12 @@ export interface BasePromptOptions {
 async function initHyperchain(envName: string, runObservability: boolean, validiumMode: boolean) {
     await announced('Initializing hyperchain creation', setupConfiguration(envName, runObservability));
     let deploymentMode = validiumMode !== undefined ? DeploymentMode.Validium : DeploymentMode.Rollup;
-    await init.initHyperCmdAction({ skipSetupCompletely: false, bumpChainId: true, runObservability, deploymentMode });
+    await init.initHyperCmdAction({
+        skipSetupCompletely: false,
+        bumpChainId: true,
+        runObservability,
+        deploymentMode
+    });
 
     // TODO: EVM:577 fix hyperchain wizard
     env.mergeInitToEnv();