From 51381cfc3c62db20d7157b0adc5868d29bb299a6 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 14:57:53 +0200 Subject: [PATCH 01/51] feat: add fees integration test to toolbox --- .../src/commands/test/args/fees.rs | 12 ++++ .../src/commands/test/args/mod.rs | 1 + .../zk_supervisor/src/commands/test/fees.rs | 62 +++++++++++++++++++ .../zk_supervisor/src/commands/test/mod.rs | 4 ++ 4 files changed, 79 insertions(+) create mode 100644 zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs create mode 100644 zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs new file mode 100644 index 000000000000..0dbd7245b11c --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs @@ -0,0 +1,12 @@ +use clap::Parser; +use serde::{Deserialize, Serialize}; + +use crate::messages::{MSG_NO_DEPS_HELP, MSG_TESTS_EXTERNAL_NODE_HELP, MSG_TEST_PATTERN_HELP}; + +#[derive(Debug, Serialize, Deserialize, Parser)] +pub struct FeesArgs { + #[clap(short, long, help = MSG_TESTS_EXTERNAL_NODE_HELP)] + pub external_node: bool, + #[clap(short, long, help = MSG_NO_DEPS_HELP)] + pub no_deps: bool, +} diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs index d74d5e64a7d5..b951608e7683 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/mod.rs @@ -1,3 +1,4 @@ +pub mod fees; pub mod integration; pub mod recovery; pub mod revert; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs new file mode 100644 index 000000000000..2370b93682ab --- /dev/null +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -0,0 +1,62 @@ +use std::path::PathBuf; + +use anyhow::Context; +use common::{cmd::Cmd, config::global_config, logger}; +use config::EcosystemConfig; +use xshell::{cmd, Shell}; + +use super::{ + args::integration::IntegrationArgs, + utils::{build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH}, +}; +use crate::messages::{ + msg_integration_tests_run, MSG_CHAIN_NOT_FOUND_ERR, MSG_DESERIALIZE_TEST_WALLETS_ERR, + MSG_INTEGRATION_TESTS_RUN_SUCCESS, +}; + +const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; + +pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + let chain_config = ecosystem_config + .load_chain(global_config().chain_name.clone()) + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); + + logger::info(msg_integration_tests_run(args.external_node)); + + if !args.no_deps { + build_contracts(shell, &ecosystem_config)?; + install_and_build_dependencies(shell, &ecosystem_config)?; + } + + let wallets_path: PathBuf = ecosystem_config.link_to_code.join(TEST_WALLETS_PATH); + let wallets: TestWallets = serde_json::from_str(shell.read_file(&wallets_path)?.as_ref()) + .context(MSG_DESERIALIZE_TEST_WALLETS_ERR)?; + + wallets + .init_test_wallet(&ecosystem_config, &chain_config) + .await?; + + let mut command = cmd!(shell, "yarn jest -- fees.test.ts") + .env("CHAIN_NAME", ecosystem_config.current_chain()) + .env("MASTER_WALLET_PK", wallets.get_test_pk(&chain_config)?); + + if args.external_node { + command = command.env("EXTERNAL_NODE", format!("{:?}", args.external_node)) + } + + if global_config().verbose { + command = command.env( + "ZKSYNC_DEBUG_LOGS", + format!("{:?}", global_config().verbose), + ) + } + + Cmd::new(command).with_force_run().run()?; + + logger::outro(MSG_INTEGRATION_TESTS_RUN_SUCCESS); + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs index 712e2f75eefd..306cc473db14 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs @@ -13,6 +13,7 @@ use crate::messages::{ mod args; mod build; +mod fees; mod integration; mod l1_contracts; mod loadtest; @@ -28,6 +29,8 @@ mod wallet; pub enum TestCommands { #[clap(about = MSG_INTEGRATION_TESTS_ABOUT, alias = "i")] Integration(IntegrationArgs), + #[clap(about = "Run fees test", alias = "i")] + Fees(IntegrationArgs), #[clap(about = MSG_REVERT_TEST_ABOUT, alias = "r")] Revert(RevertArgs), #[clap(about = MSG_RECOVERY_TEST_ABOUT, alias = "rec")] @@ -51,6 +54,7 @@ pub enum TestCommands { pub async fn run(shell: &Shell, args: TestCommands) -> anyhow::Result<()> { match args { TestCommands::Integration(args) => integration::run(shell, args).await, + TestCommands::Fees(args) => fees::run(shell, args).await, TestCommands::Revert(args) => revert::run(shell, args).await, TestCommands::Recovery(args) => recovery::run(shell, args).await, TestCommands::Upgrade(args) => upgrade::run(shell, args), From 19b097656481118e9e62c4f0e16c9d003bdd7e30 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 15:08:32 +0200 Subject: [PATCH 02/51] ci: add CI step for fees integration test --- .github/workflows/ci-zk-toolbox-reusable.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index 98a44c443581..144084523994 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -309,6 +309,9 @@ jobs: wait $PID3 wait $PID4 + - name: Fee projection tests + run: ci_run zk_supervisor test fees + - name: Run revert tests run: | ci_run killall -INT zksync_server || true From 4f586d50060c467f899d38309bd386c85b92c861 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 15:14:15 +0200 Subject: [PATCH 03/51] ci: add CI step for fees integration test --- .github/workflows/ci-core-reusable.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index cac585a00a87..f8fe63df2d49 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -444,6 +444,9 @@ jobs: wait $PID3 wait $PID4 + - name: Fee projection tests + run: ci_run zk_supervisor test fees + - name: Run revert tests run: | ci_run killall -INT zksync_server || true From 0d01aa63f24797ab142a3e570a289249555a0017 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 15:27:24 +0200 Subject: [PATCH 04/51] refactor: use a single constant for TS_INTEGRATION_PATH --- contracts | 2 +- zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs | 7 ++++--- .../crates/zk_supervisor/src/commands/test/integration.rs | 7 ++++--- zk_toolbox/crates/zk_supervisor/src/commands/test/utils.rs | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/contracts b/contracts index bce4b2d0f34b..3a1b5d4b94ff 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit bce4b2d0f34bd87f1aaadd291772935afb1c3bd6 +Subproject commit 3a1b5d4b94ffb00f03d436a7db7e48589eb74d39 diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index 2370b93682ab..b87d40d040be 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -7,15 +7,16 @@ use xshell::{cmd, Shell}; use super::{ args::integration::IntegrationArgs, - utils::{build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH}, + utils::{ + build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH, + TS_INTEGRATION_PATH, + }, }; use crate::messages::{ msg_integration_tests_run, MSG_CHAIN_NOT_FOUND_ERR, MSG_DESERIALIZE_TEST_WALLETS_ERR, MSG_INTEGRATION_TESTS_RUN_SUCCESS, }; -const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; - pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/integration.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/integration.rs index fc3ba974118c..d7e11fb25f0e 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/integration.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/integration.rs @@ -7,15 +7,16 @@ use xshell::{cmd, Shell}; use super::{ args::integration::IntegrationArgs, - utils::{build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH}, + utils::{ + build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH, + TS_INTEGRATION_PATH, + }, }; use crate::messages::{ msg_integration_tests_run, MSG_CHAIN_NOT_FOUND_ERR, MSG_DESERIALIZE_TEST_WALLETS_ERR, MSG_INTEGRATION_TESTS_RUN_SUCCESS, }; -const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; - pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/utils.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/utils.rs index 3a5cfd179cc4..8656ff44d319 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/utils.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/utils.rs @@ -17,7 +17,7 @@ use crate::messages::{ pub const TEST_WALLETS_PATH: &str = "etc/test_config/constant/eth.json"; const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; -const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; +pub const TS_INTEGRATION_PATH: &str = "core/tests/ts-integration"; const CONTRACTS_TEST_DATA_PATH: &str = "etc/contracts-test-data"; #[derive(Deserialize)] From f9343847fe621f68e1849f74c23ba8c65bcb310d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 15:35:16 +0200 Subject: [PATCH 05/51] style: lint --- zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs index 0dbd7245b11c..6f1942c8bff5 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs @@ -1,7 +1,7 @@ use clap::Parser; use serde::{Deserialize, Serialize}; -use crate::messages::{MSG_NO_DEPS_HELP, MSG_TESTS_EXTERNAL_NODE_HELP, MSG_TEST_PATTERN_HELP}; +use crate::messages::{MSG_NO_DEPS_HELP, MSG_TESTS_EXTERNAL_NODE_HELP}; #[derive(Debug, Serialize, Deserialize, Parser)] pub struct FeesArgs { From 3028221cba848d2b940c94bad65c4aba305e1544 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 16:40:31 +0200 Subject: [PATCH 06/51] refactor: simplify code --- .../src/commands/test/args/fees.rs | 4 +-- .../zk_supervisor/src/commands/test/fees.rs | 27 +++---------------- 2 files changed, 4 insertions(+), 27 deletions(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs index 6f1942c8bff5..c6a5db8dd292 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs @@ -1,12 +1,10 @@ use clap::Parser; use serde::{Deserialize, Serialize}; -use crate::messages::{MSG_NO_DEPS_HELP, MSG_TESTS_EXTERNAL_NODE_HELP}; +use crate::messages::MSG_NO_DEPS_HELP; #[derive(Debug, Serialize, Deserialize, Parser)] pub struct FeesArgs { - #[clap(short, long, help = MSG_TESTS_EXTERNAL_NODE_HELP)] - pub external_node: bool, #[clap(short, long, help = MSG_NO_DEPS_HELP)] pub no_deps: bool, } diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index b87d40d040be..14c017445019 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -1,28 +1,16 @@ -use std::path::PathBuf; - -use anyhow::Context; use common::{cmd::Cmd, config::global_config, logger}; use config::EcosystemConfig; use xshell::{cmd, Shell}; use super::{ args::integration::IntegrationArgs, - utils::{ - build_contracts, install_and_build_dependencies, TestWallets, TEST_WALLETS_PATH, - TS_INTEGRATION_PATH, - }, -}; -use crate::messages::{ - msg_integration_tests_run, MSG_CHAIN_NOT_FOUND_ERR, MSG_DESERIALIZE_TEST_WALLETS_ERR, - MSG_INTEGRATION_TESTS_RUN_SUCCESS, + utils::{build_contracts, install_and_build_dependencies, TS_INTEGRATION_PATH}, }; +use crate::messages::{msg_integration_tests_run, MSG_INTEGRATION_TESTS_RUN_SUCCESS}; pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; - let chain_config = ecosystem_config - .load_chain(global_config().chain_name.clone()) - .context(MSG_CHAIN_NOT_FOUND_ERR)?; shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); logger::info(msg_integration_tests_run(args.external_node)); @@ -32,17 +20,8 @@ pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { install_and_build_dependencies(shell, &ecosystem_config)?; } - let wallets_path: PathBuf = ecosystem_config.link_to_code.join(TEST_WALLETS_PATH); - let wallets: TestWallets = serde_json::from_str(shell.read_file(&wallets_path)?.as_ref()) - .context(MSG_DESERIALIZE_TEST_WALLETS_ERR)?; - - wallets - .init_test_wallet(&ecosystem_config, &chain_config) - .await?; - let mut command = cmd!(shell, "yarn jest -- fees.test.ts") - .env("CHAIN_NAME", ecosystem_config.current_chain()) - .env("MASTER_WALLET_PK", wallets.get_test_pk(&chain_config)?); + .env("CHAIN_NAME", ecosystem_config.current_chain()); if args.external_node { command = command.env("EXTERNAL_NODE", format!("{:?}", args.external_node)) From 4379a32d0b4c217d4f6fad7451448027d0546f92 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 18:02:03 +0200 Subject: [PATCH 07/51] fix: restore contracts --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index 3a1b5d4b94ff..bce4b2d0f34b 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 3a1b5d4b94ffb00f03d436a7db7e48589eb74d39 +Subproject commit bce4b2d0f34bd87f1aaadd291772935afb1c3bd6 From 732ab9d4bc3d7bf5c56eb6b534f19ded1de760e3 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 17 Sep 2024 18:02:32 +0200 Subject: [PATCH 08/51] feat: test fees on all chains --- .github/workflows/ci-core-reusable.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 26785ef08339..010da8821bc1 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -445,7 +445,11 @@ jobs: wait $PID4 - name: Fee projection tests - run: ci_run zk_supervisor test fees + run: | + ci_run zk_supervisor test fees --chain era + ci_run zk_supervisor test fees --chain validium + ci_run zk_supervisor test fees --chain custom_token + ci_run zk_supervisor test fees --chain consensus - name: Run revert tests run: | From 5b816e14b710f6c56d2733f72618452ab2828289 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 09:50:26 +0200 Subject: [PATCH 09/51] refactor: remove unused arg --- .../zk_supervisor/src/commands/test/fees.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index 14c017445019..8afebf1bc83f 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -3,30 +3,29 @@ use config::EcosystemConfig; use xshell::{cmd, Shell}; use super::{ - args::integration::IntegrationArgs, + args::fees::FeesArgs, utils::{build_contracts, install_and_build_dependencies, TS_INTEGRATION_PATH}, }; use crate::messages::{msg_integration_tests_run, MSG_INTEGRATION_TESTS_RUN_SUCCESS}; -pub async fn run(shell: &Shell, args: IntegrationArgs) -> anyhow::Result<()> { +pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; - shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); - logger::info(msg_integration_tests_run(args.external_node)); - if !args.no_deps { + logger::info("Installing dependencies"); build_contracts(shell, &ecosystem_config)?; install_and_build_dependencies(shell, &ecosystem_config)?; } + logger::info(format!( + "Running fees tests on chain: {}", + ecosystem_config.current_chain() + )); + let mut command = cmd!(shell, "yarn jest -- fees.test.ts") .env("CHAIN_NAME", ecosystem_config.current_chain()); - if args.external_node { - command = command.env("EXTERNAL_NODE", format!("{:?}", args.external_node)) - } - if global_config().verbose { command = command.env( "ZKSYNC_DEBUG_LOGS", From c364c8e8410b07fd3ca2dc684dcc2ed0a3e7bb8d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 09:50:45 +0200 Subject: [PATCH 10/51] ci: parallelize fees testing --- .github/workflows/ci-core-reusable.yml | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 010da8821bc1..1649134c17e7 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -175,6 +175,7 @@ jobs: SNAPSHOT_RECOVERY_LOGS_DIR=logs/snapshot_recovery/ GENESIS_RECOVERY_LOGS_DIR=logs/genesis_recovery/ EXTERNAL_NODE_LOGS_DIR=logs/external_node + FEES_LOGS_DIR=logs/fees REVERT_LOGS_DIR=logs/revert mkdir -p $SERVER_LOGS_DIR @@ -183,6 +184,7 @@ jobs: mkdir -p $SNAPSHOT_RECOVERY_LOGS_DIR mkdir -p $GENESIS_RECOVERY_LOGS_DIR mkdir -p $EXTERNAL_NODE_LOGS_DIR + mkdir -p $FEES_LOGS_DIR mkdir -p $REVERT_LOGS_DIR echo "SERVER_LOGS_DIR=$SERVER_LOGS_DIR" >> $GITHUB_ENV @@ -446,10 +448,23 @@ jobs: - name: Fee projection tests run: | - ci_run zk_supervisor test fees --chain era - ci_run zk_supervisor test fees --chain validium - ci_run zk_supervisor test fees --chain custom_token - ci_run zk_supervisor test fees --chain consensus + ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log & + PID1=$! + + ci_run zk_supervisor test fees --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log & + PID2=$! + + ci_run zk_supervisor test fees --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log & + PID3=$! + + ci_run zk_supervisor test fees --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log & + PID4=$! + + wait $PID1 + wait $PID2 + wait $PID3 + wait $PID4 + - name: Run revert tests run: | From c67a0f1f95ea2cf3367660e3c924c0a27077c13f Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 09:52:15 +0200 Subject: [PATCH 11/51] style: remove unused import --- zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index 8afebf1bc83f..e9808eef5a6d 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -6,7 +6,7 @@ use super::{ args::fees::FeesArgs, utils::{build_contracts, install_and_build_dependencies, TS_INTEGRATION_PATH}, }; -use crate::messages::{msg_integration_tests_run, MSG_INTEGRATION_TESTS_RUN_SUCCESS}; +use crate::messages::MSG_INTEGRATION_TESTS_RUN_SUCCESS; pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; From aeb9c5073a1f992dee9a674eb26b83af7368ffd5 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 09:56:10 +0200 Subject: [PATCH 12/51] fix: wrong args type --- zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs index 152b72e0a472..ae6b4518e6db 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/mod.rs @@ -1,6 +1,6 @@ use args::{ - integration::IntegrationArgs, recovery::RecoveryArgs, revert::RevertArgs, rust::RustArgs, - upgrade::UpgradeArgs, + fees::FeesArgs, integration::IntegrationArgs, recovery::RecoveryArgs, revert::RevertArgs, + rust::RustArgs, upgrade::UpgradeArgs, }; use clap::Subcommand; use xshell::Shell; @@ -31,7 +31,7 @@ pub enum TestCommands { #[clap(about = MSG_INTEGRATION_TESTS_ABOUT, alias = "i")] Integration(IntegrationArgs), #[clap(about = "Run fees test", alias = "i")] - Fees(IntegrationArgs), + Fees(FeesArgs), #[clap(about = MSG_REVERT_TEST_ABOUT, alias = "r")] Revert(RevertArgs), #[clap(about = MSG_RECOVERY_TEST_ABOUT, alias = "rec")] From 9b363d6820b0be580889974f69f7cbbac6461e48 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 11:02:33 +0200 Subject: [PATCH 13/51] feat: add FEES_LOGS_DIR to GITHUB_ENV --- .github/workflows/ci-core-reusable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 1649134c17e7..94cc8809d08e 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -193,6 +193,7 @@ jobs: echo "SNAPSHOT_RECOVERY_LOGS_DIR=$SNAPSHOT_RECOVERY_LOGS_DIR" >> $GITHUB_ENV echo "GENESIS_RECOVERY_LOGS_DIR=$GENESIS_RECOVERY_LOGS_DIR" >> $GITHUB_ENV echo "EXTERNAL_NODE_LOGS_DIR=$EXTERNAL_NODE_LOGS_DIR" >> $GITHUB_ENV + echo "FEES_LOGS_DIR=$FEES_LOGS_DIR" >> $GITHUB_ENV echo "REVERT_LOGS_DIR=$REVERT_LOGS_DIR" >> $GITHUB_ENV - name: Initialize ecosystem From 8c87f51de92d010c6e1033332c573cbb0844f88c Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 16:40:28 +0200 Subject: [PATCH 14/51] fix: truly run fees testing --- zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index e9808eef5a6d..6eeba98442db 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -24,6 +24,7 @@ pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { )); let mut command = cmd!(shell, "yarn jest -- fees.test.ts") + .env("RUN_FEE_TEST", "1") .env("CHAIN_NAME", ecosystem_config.current_chain()); if global_config().verbose { From b270415a206a1f75592fb0be70e671f60212ade8 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 16:42:46 +0200 Subject: [PATCH 15/51] revert: temporarily revert to sequential fees testing --- .github/workflows/ci-core-reusable.yml | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 94cc8809d08e..bb05a3318335 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -449,24 +449,11 @@ jobs: - name: Fee projection tests run: | - ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log & - PID1=$! - - ci_run zk_supervisor test fees --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log & - PID2=$! - - ci_run zk_supervisor test fees --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log & - PID3=$! - - ci_run zk_supervisor test fees --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log & - PID4=$! - - wait $PID1 - wait $PID2 - wait $PID3 - wait $PID4 + ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log + ci_run zk_supervisor test fees --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log + ci_run zk_supervisor test fees --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log + ci_run zk_supervisor test fees --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log - - name: Run revert tests run: | ci_run killall -INT zksync_server || true From 0b29ee49670e04578be9db971741fb6dd2553d0d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Wed, 18 Sep 2024 17:56:55 +0200 Subject: [PATCH 16/51] fix: try to re-run servers before fees testing --- .github/workflows/ci-core-reusable.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index bb05a3318335..303ad370e3cd 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -447,6 +447,15 @@ jobs: wait $PID3 wait $PID4 + - name: Re-run servers + run: | + ci_run zk_inception server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log & + ci_run zk_inception server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log & + ci_run zk_inception server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log & + ci_run zk_inception server --ignore-prerequisites --chain consensus \ + --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \ + &> ${{ env.SERVER_LOGS_DIR }}/consensus.log & + - name: Fee projection tests run: | ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log From 2860b8f071c072233bddcb4d9c838046b2342c83 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Thu, 19 Sep 2024 09:54:45 +0200 Subject: [PATCH 17/51] revert: do not re-run servers --- .github/workflows/ci-core-reusable.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 303ad370e3cd..bb05a3318335 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -447,15 +447,6 @@ jobs: wait $PID3 wait $PID4 - - name: Re-run servers - run: | - ci_run zk_inception server --ignore-prerequisites --chain era &> ${{ env.SERVER_LOGS_DIR }}/rollup.log & - ci_run zk_inception server --ignore-prerequisites --chain validium &> ${{ env.SERVER_LOGS_DIR }}/validium.log & - ci_run zk_inception server --ignore-prerequisites --chain custom_token &> ${{ env.SERVER_LOGS_DIR }}/custom_token.log & - ci_run zk_inception server --ignore-prerequisites --chain consensus \ - --components=api,tree,eth,state_keeper,housekeeper,commitment_generator,vm_runner_protective_reads,vm_runner_bwip,vm_playground,da_dispatcher,consensus \ - &> ${{ env.SERVER_LOGS_DIR }}/consensus.log & - - name: Fee projection tests run: | ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log From ebbe4c1bf7d42ba27aaa6cedb68bb356656406b1 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Thu, 19 Sep 2024 10:31:51 +0200 Subject: [PATCH 18/51] style: remove optional double dash --- zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index 6eeba98442db..99f48aa33a09 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -23,7 +23,7 @@ pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { ecosystem_config.current_chain() )); - let mut command = cmd!(shell, "yarn jest -- fees.test.ts") + let mut command = cmd!(shell, "yarn jest fees.test.ts") .env("RUN_FEE_TEST", "1") .env("CHAIN_NAME", ecosystem_config.current_chain()); From 66c4f5fc1dea6b8099f1cf7c05f8b2c85bee03b8 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Thu, 19 Sep 2024 16:45:59 +0200 Subject: [PATCH 19/51] refactor: use managed start/kill node --- .../src/jest-setup/global-setup.ts | 42 ++++ core/tests/ts-integration/src/tester.ts | 125 ++++++++++ core/tests/ts-integration/src/utils.ts | 217 ++++++++++++++++++ core/tests/ts-integration/tests/fees.test.ts | 193 ++++++++-------- 4 files changed, 475 insertions(+), 102 deletions(-) create mode 100644 core/tests/ts-integration/src/tester.ts create mode 100644 core/tests/ts-integration/src/utils.ts diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index d84d70fe69da..3b343ed91276 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -1,5 +1,10 @@ +import { shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { bigIntReplacer } from '../helpers'; import { TestContextOwner, loadTestEnvironment } from '../index'; +import path from 'path'; +import { runServerInBackground } from '../utils'; +import utils, { sleep } from 'utils'; +import { killPidWithAllChilds } from 'utils/build/kill'; declare global { var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; @@ -19,11 +24,48 @@ async function performSetup(_globalConfig: any, _projectConfig: any) { // Before starting any actual logic, we need to ensure that the server is running (it may not // be the case, for example, right after deployment on stage). + const fileConfig = shouldLoadConfigFromFile(); + const pathToHome = path.join(__dirname, '../../../../..'); + const autoKill: boolean = !fileConfig.loadFromFile || !process.env.NO_KILL; + + if (autoKill) { + try { + await utils.exec(`killall -KILL zksync_server`); + } catch (err) { + console.log(`ignored error: ${err}`); + } + } + + let components = 'api,tree,eth,state_keeper,da_dispatcher,vm_runner_protective_reads'; + const env = process.env; + + let proc = runServerInBackground({ + components: [components], + stdio: ['ignore', 'ignore', 'ignore'], + cwd: pathToHome, + env: env, + useZkInception: fileConfig.loadFromFile, + chain: fileConfig.chain + }); const testEnvironment = await loadTestEnvironment(); const testContextOwner = new TestContextOwner(testEnvironment); const testContext = await testContextOwner.setupContext(); + // TODO: add PID to context and let the test suites kill the server. + try { + await killPidWithAllChilds(proc.pid!, 9); + } catch (err) { + console.log(`ignored error: ${err}`); + } + + let iter = 0; + while (iter < 30) { + console.log('Waiting for the server to stop...'); + await sleep(1); + iter += 1; + } + // Set the test context for test suites to pick up. // Currently, jest doesn't provide a way to pass data from `globalSetup` to suites, // so we store the data as serialized JSON. diff --git a/core/tests/ts-integration/src/tester.ts b/core/tests/ts-integration/src/tester.ts new file mode 100644 index 000000000000..1809b4c2784c --- /dev/null +++ b/core/tests/ts-integration/src/tester.ts @@ -0,0 +1,125 @@ +import { expect } from 'chai'; +import * as ethers from 'ethers'; +import * as zksync from 'zksync-ethers'; +import * as fs from 'fs'; +import * as path from 'path'; + +const BASE_ERC20_TO_MINT = ethers.parseEther('100'); + +export class Tester { + public runningFee: Map; + + constructor( + public ethProvider: ethers.Provider, + public ethWallet: ethers.Wallet, + public syncWallet: zksync.Wallet, + public web3Provider: zksync.Provider, + public isETHBasedChain: boolean, + public baseTokenAddress: string + ) { + this.runningFee = new Map(); + } + + // prettier-ignore + static async init(l1_rpc_addr: string, l2_rpc_addr: string, baseTokenAddress: string): Promise { + const ethProvider = new ethers.JsonRpcProvider(l1_rpc_addr); + ethProvider.pollingInterval = 100; + + const testConfigPath = path.join(process.env.ZKSYNC_HOME!, `etc/test_config/constant`); + const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, {encoding: 'utf-8'})); + + let ethWalletPK: string; + if (process.env.MASTER_WALLET_PK) { + ethWalletPK = process.env.MASTER_WALLET_PK; + } else { + const ethWalletHD = ethers.HDNodeWallet.fromMnemonic( + ethers.Mnemonic.fromPhrase(ethTestConfig.test_mnemonic), + "m/44'/60'/0'/0/0" + ); + + ethWalletPK = ethWalletHD.privateKey + } + + const ethWallet = new ethers.Wallet(ethWalletPK, ethProvider); + + const web3Provider = new zksync.Provider(l2_rpc_addr); + web3Provider.pollingInterval = 100; // It's OK to keep it low even on stage. + const syncWallet = new zksync.Wallet(ethWallet.privateKey, web3Provider, ethProvider); + + + // Since some tx may be pending on stage, we don't want to get stuck because of it. + // In order to not get stuck transactions, we manually cancel all the pending txs. + const latestNonce = await ethWallet.getNonce('latest'); + const pendingNonce = await ethWallet.getNonce('pending'); + const cancellationTxs = []; + for (let nonce = latestNonce; nonce != pendingNonce; nonce++) { + // For each transaction to override it, we need to provide greater fee. + // We would manually provide a value high enough (for a testnet) to be both valid + // and higher than the previous one. It's OK as we'll only be charged for the bass fee + // anyways. We will also set the miner's tip to 5 gwei, which is also much higher than the normal one. + const maxFeePerGas = ethers.parseEther("0.00000025"); // 250 gwei + const maxPriorityFeePerGas = ethers.parseEther("0.000000005"); // 5 gwei + cancellationTxs.push(ethWallet.sendTransaction({ + to: ethWallet.address, + nonce, + maxFeePerGas, + maxPriorityFeePerGas + }).then((tx) => tx.wait())); + } + if (cancellationTxs.length > 0) { + await Promise.all(cancellationTxs); + console.log(`Canceled ${cancellationTxs.length} pending transactions`); + } + + const isETHBasedChain = baseTokenAddress == zksync.utils.ETH_ADDRESS_IN_CONTRACTS; + + return new Tester(ethProvider, ethWallet, syncWallet, web3Provider, isETHBasedChain, baseTokenAddress); + } + + /// Ensures that the main wallet has enough base token. + /// This can not be done inside the `init` function because `init` function can be called before the + /// L2 RPC is active, but we need the L2 RPC to get the base token address. + async fundSyncWallet() { + const baseTokenAddress = await this.syncWallet.provider.getBaseTokenContractAddress(); + if (!(baseTokenAddress === zksync.utils.ETH_ADDRESS_IN_CONTRACTS)) { + const l1Erc20ABI = ['function mint(address to, uint256 amount)']; + const l1Erc20Contract = new ethers.Contract(baseTokenAddress, l1Erc20ABI, this.ethWallet); + await (await l1Erc20Contract.mint(this.ethWallet.address, BASE_ERC20_TO_MINT)).wait(); + } + } + + async fundedWallet(ethAmount: bigint, l1Token: zksync.types.Address, tokenAmount: bigint) { + const newWalletHD = zksync.Wallet.createRandom(); + const newWallet = new zksync.Wallet(newWalletHD.privateKey, this.web3Provider, this.ethProvider); + + let ethBalance = await this.syncWallet.getBalanceL1(); + expect(ethBalance > ethAmount, 'Insufficient eth balance to create funded wallet').to.be.true; + + // To make the wallet capable of requesting priority operations, + // send ETH to L1. + + const tx1 = await this.syncWallet.ethWallet().sendTransaction({ + to: newWallet.address, + value: ethAmount + }); + await tx1.wait(); + + // Funds the wallet with L1 token. + + let tokenBalance = await this.syncWallet.getBalanceL1(l1Token); + expect(tokenBalance > tokenAmount, 'Insufficient token balance to create funded wallet').to.be.true; + + const erc20ABI = ['function transfer(address to, uint256 amount)']; + const erc20Contract = new ethers.Contract(l1Token, erc20ABI, this.ethWallet); + + const tx2 = await erc20Contract.transfer(newWallet.address, tokenAmount); + await tx2.wait(); + + return newWallet; + } + + emptyWallet() { + const walletHD = zksync.Wallet.createRandom(); + return new zksync.Wallet(walletHD.privateKey, this.web3Provider, this.ethProvider); + } +} diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts new file mode 100644 index 000000000000..bc4830bb3fcf --- /dev/null +++ b/core/tests/ts-integration/src/utils.ts @@ -0,0 +1,217 @@ +import { exec as _exec, spawn as _spawn, ChildProcessWithoutNullStreams, type ProcessEnvOptions } from 'child_process'; +import { assert } from 'chai'; +import { FileConfig } from 'utils/build/file-configs'; +import { Tester } from './tester'; +import { killPidWithAllChilds } from 'utils/build/kill'; +import * as utils from 'utils'; +import fs from 'node:fs/promises'; +import * as zksync from 'zksync-ethers'; + +// executes a command in background and returns a child process handle +// by default pipes data to parent's stdio but this can be overridden +export function background({ + command, + stdio = 'inherit', + cwd, + env +}: { + command: string; + stdio: any; + cwd?: ProcessEnvOptions['cwd']; + env?: ProcessEnvOptions['env']; +}): ChildProcessWithoutNullStreams { + command = command.replace(/\n/g, ' '); + console.log(`Run command ${command}`); + return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env }); +} + +export function runInBackground({ + command, + components, + stdio, + cwd, + env +}: { + command: string; + components?: string[]; + stdio: any; + cwd?: Parameters[0]['cwd']; + env?: Parameters[0]['env']; +}): ChildProcessWithoutNullStreams { + if (components && components.length > 0) { + command += ` --components=${components.join(',')}`; + } + return background({ command, stdio, cwd, env }); +} + +export function runServerInBackground({ + components, + stdio, + cwd, + env, + useZkInception, + newL1GasPrice, + newPubdataPrice, + chain +}: { + components?: string[]; + stdio: any; + cwd?: Parameters[0]['cwd']; + env?: Parameters[0]['env']; + useZkInception?: boolean; + newL1GasPrice?: string; + newPubdataPrice?: string; + chain?: string; +}): ChildProcessWithoutNullStreams { + let command = ''; + if (useZkInception) { + command = 'zk_inception server'; + if (chain) { + command += ` --chain ${chain}`; + } + } else { + command = 'zk server'; + command = `DATABASE_MERKLE_TREE_MODE=full ${command}`; + + if (newPubdataPrice) { + command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_PUBDATA_PRICE=${newPubdataPrice} ${command}`; + } + + if (newL1GasPrice) { + // We need to ensure that each transaction gets into its own batch for more fair comparison. + command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_L1_GAS_PRICE=${newL1GasPrice} ${command}`; + } + + const testMode = newPubdataPrice || newL1GasPrice; + if (testMode) { + // We need to ensure that each transaction gets into its own batch for more fair comparison. + command = `CHAIN_STATE_KEEPER_TRANSACTION_SLOTS=1 ${command}`; + } + } + return runInBackground({ command, components, stdio, cwd, env }); +} + +export interface MainNodeSpawnOptions { + enableConsensus: boolean; + ethClientWeb3Url: string; + apiWeb3JsonRpcHttpUrl: string; + baseTokenAddress: string; +} + +export enum NodeType { + MAIN = 'zksync_server', + EXT = 'zksync_external_node' +} + +export class Node { + constructor( + public readonly tester: Tester, + private readonly proc: ChildProcessWithoutNullStreams, + private readonly type: TYPE + ) { } + + public async terminate() { + try { + await killPidWithAllChilds(this.proc.pid!, 9); + } catch (err) { + console.log(`ignored error: ${err}`); + } + } + + /** + * Terminates all main node processes running. + * + * WARNING: This is not safe to use when running nodes on multiple chains. + */ + public static async killAll(type: NodeType) { + try { + await utils.exec(`killall -KILL ${type}`); + } catch (err) { + console.log(`ignored error: ${err}`); + } + } + + /** Waits for the node process to exit. */ + public async waitForExit(): Promise { + while (this.proc.exitCode === null) { + await utils.sleep(1); + } + return this.proc.exitCode; + } + + public async killAndWaitForShutdown() { + await this.terminate(); + // Wait until it's really stopped. + let iter = 0; + while (iter < 30) { + try { + await this.tester.syncWallet.provider.getBlockNumber(); + await utils.sleep(2); + iter += 1; + } catch (_) { + // When exception happens, we assume that server died. + return; + } + } + // It's going to panic anyway, since the server is a singleton entity, so better to exit early. + throw new Error(`${this.type} didn't stop after a kill request`); + } +} + +export class NodeSpawner { + public constructor( + private readonly pathToHome: string, + private readonly logs: fs.FileHandle, + private readonly fileConfig: FileConfig, + private readonly options: MainNodeSpawnOptions, + private readonly env?: ProcessEnvOptions['env'] + ) { } + + public async spawnMainNode(newL1GasPrice?: string, newPubdataPrice?: string): Promise> { + const env = this.env ?? process.env; + const { fileConfig, pathToHome, options, logs } = this; + + let components = 'api,tree,eth,state_keeper,da_dispatcher,vm_runner_protective_reads'; + if (options.enableConsensus) { + components += ',consensus'; + } + if (options.baseTokenAddress != zksync.utils.LEGACY_ETH_ADDRESS) { + components += ',base_token_ratio_persister'; + } + let proc = runServerInBackground({ + components: [components], + stdio: ['ignore', logs, logs], + cwd: pathToHome, + env: env, + useZkInception: fileConfig.loadFromFile, + newL1GasPrice: newL1GasPrice, + newPubdataPrice: newPubdataPrice, + chain: fileConfig.chain + }); + + // Wait until the main node starts responding. + const tester = await Tester.init( + options.ethClientWeb3Url, + options.apiWeb3JsonRpcHttpUrl, + options.baseTokenAddress + ); + await waitForNodeToStart(tester, proc, options.apiWeb3JsonRpcHttpUrl); + return new Node(tester, proc, NodeType.MAIN); + } +} + +async function waitForNodeToStart(tester: Tester, proc: ChildProcessWithoutNullStreams, l2Url: string) { + while (true) { + try { + const blockNumber = await tester.syncWallet.provider.getBlockNumber(); + console.log(`Initialized node API on ${l2Url}; latest block: ${blockNumber}`); + break; + } catch (err) { + if (proc.exitCode != null) { + assert.fail(`server failed to start, exitCode = ${proc.exitCode}`); + } + console.log(`Node waiting for API on ${l2Url}`); + await utils.sleep(1); + } + } +} diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 8d5b7a23a94d..80122b07ba16 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -10,59 +10,115 @@ * */ import * as utils from 'utils'; -import * as fs from 'fs'; +import fs from 'node:fs/promises'; import { TestMaster } from '../src'; import * as zksync from 'zksync-ethers'; import * as ethers from 'ethers'; import { DataAvailabityMode, Token } from '../src/types'; import { SYSTEM_CONTEXT_ADDRESS, getTestContract } from '../src/helpers'; +import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs'; +import { logsTestPath } from 'utils/build/logs'; +import path from 'path'; +import { NodeSpawner, Node, NodeType } from '../src/utils'; const UINT32_MAX = 2n ** 32n - 1n; const MAX_GAS_PER_PUBDATA = 50_000n; -const logs = fs.createWriteStream('fees.log', { flags: 'a' }); - -// Unless `RUN_FEE_TEST` is provided, skip the test suit -const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; - // The L1 gas prices under which the test will be conducted. // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; - -testFees('Test fees', () => { + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; + +// Unless `RUN_FEE_TEST` is provided, skip the test suit +const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; + +testFees('Test fees', function () { let testMaster: TestMaster; let alice: zksync.Wallet; let tokenDetails: Token; let aliceErc20: zksync.Contract; - beforeAll(() => { + let mainLogs: fs.FileHandle; + let baseTokenAddress: string; + let ethClientWeb3Url: string; + let apiWeb3JsonRpcHttpUrl: string; + let mainNodeSpawner: NodeSpawner; + let mainNode: Node; + + const fileConfig = shouldLoadConfigFromFile(); + const pathToHome = path.join(__dirname, '../../../..'); + const enableConsensus = process.env.ENABLE_CONSENSUS == 'true'; + + async function logsPath(name: string): Promise { + return await logsTestPath(fileConfig.chain, 'logs/revert/', name); + } + + beforeAll(async () => { testMaster = TestMaster.getInstance(__filename); alice = testMaster.mainAccount(); tokenDetails = testMaster.environment().erc20Token; aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); + + if (!fileConfig.loadFromFile) { + ethClientWeb3Url = process.env.ETH_CLIENT_WEB3_URL!; + apiWeb3JsonRpcHttpUrl = process.env.API_WEB3_JSON_RPC_HTTP_URL!; + baseTokenAddress = process.env.CONTRACTS_BASE_TOKEN_ADDR!; + } else { + const generalConfig = loadConfig({ + pathToHome, + chain: fileConfig.chain, + config: 'general.yaml' + }); + const secretsConfig = loadConfig({ + pathToHome, + chain: fileConfig.chain, + config: 'secrets.yaml' + }); + const contractsConfig = loadConfig({ + pathToHome, + chain: fileConfig.chain, + config: 'contracts.yaml' + }); + + ethClientWeb3Url = secretsConfig.l1.l1_rpc_url; + apiWeb3JsonRpcHttpUrl = generalConfig.api.web3_json_rpc.http_url; + baseTokenAddress = contractsConfig.l1.base_token_addr; + } + + const pathToMainLogs = await logsPath('server.log'); + mainLogs = await fs.open(pathToMainLogs, 'a'); + console.log(`Writing server logs to ${pathToMainLogs}`); + + mainNodeSpawner = new NodeSpawner(pathToHome, mainLogs, fileConfig, { + enableConsensus, + ethClientWeb3Url, + apiWeb3JsonRpcHttpUrl, + baseTokenAddress + }); }); test('Test fees', async () => { + mainNode = await mainNodeSpawner.spawnMainNode(); + const receiver = ethers.Wallet.createRandom().address; // Getting ETH price in gas. @@ -110,6 +166,10 @@ testFees('Test fees', () => { 'ERC20 transfer (to old):\n\n' ]; for (const gasPrice of L1_GAS_PRICES_TO_TEST) { + // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode(gasPrice.toString(), gasPrice.toString()); + reports = await appendResults( alice, [feeTestL1Receipt, feeTestL1Receipt, feeTestL1ReceiptERC20, feeTestL1ReceiptERC20], @@ -144,6 +204,9 @@ testFees('Test fees', () => { }); test('Test gas consumption under large L1 gas price', async () => { + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode(); + if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. return; @@ -163,8 +226,8 @@ testFees('Test fees', () => { // that the gasLimit is indeed over u32::MAX, which is the most important tested property. const requiredPubdataPrice = minimalL2GasPrice * 100_000n; - await setInternalL1GasPrice( - alice._providerL2(), + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode( requiredPubdataPrice.toString(), requiredPubdataPrice.toString() ); @@ -208,7 +271,8 @@ testFees('Test fees', () => { afterAll(async () => { // Returning the pubdata price to the default one - await setInternalL1GasPrice(alice._providerL2(), undefined, undefined, true); + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode(); await testMaster.deinitialize(); }); @@ -221,9 +285,6 @@ async function appendResults( newL1GasPrice: bigint, reports: string[] ): Promise { - // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. - await setInternalL1GasPrice(sender._providerL2(), newL1GasPrice.toString(), newL1GasPrice.toString()); - if (originalL1Receipts.length !== reports.length && originalL1Receipts.length !== transactionRequests.length) { throw new Error('The array of receipts and reports have different length'); } @@ -274,75 +335,3 @@ async function updateReport( return oldReport + gasReport; } - -async function killServerAndWaitForShutdown(provider: zksync.Provider) { - await utils.exec('pkill zksync_server'); - // Wait until it's really stopped. - let iter = 0; - while (iter < 30) { - try { - await provider.getBlockNumber(); - await utils.sleep(2); - iter += 1; - } catch (_) { - // When exception happens, we assume that server died. - return; - } - } - // It's going to panic anyway, since the server is a singleton entity, so better to exit early. - throw new Error("Server didn't stop after a kill request"); -} - -async function setInternalL1GasPrice( - provider: zksync.Provider, - newL1GasPrice?: string, - newPubdataPrice?: string, - disconnect?: boolean -) { - // Make sure server isn't running. - try { - await killServerAndWaitForShutdown(provider); - } catch (_) {} - - // Run server in background. - let command = 'zk server --components api,tree,eth,state_keeper,da_dispatcher,vm_runner_protective_reads'; - command = `DATABASE_MERKLE_TREE_MODE=full ${command}`; - - if (newPubdataPrice) { - command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_PUBDATA_PRICE=${newPubdataPrice} ${command}`; - } - - if (newL1GasPrice) { - // We need to ensure that each transaction gets into its own batch for more fair comparison. - command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_L1_GAS_PRICE=${newL1GasPrice} ${command}`; - } - - const testMode = newPubdataPrice || newL1GasPrice; - if (testMode) { - // We need to ensure that each transaction gets into its own batch for more fair comparison. - command = `CHAIN_STATE_KEEPER_TRANSACTION_SLOTS=1 ${command}`; - } - - const zkSyncServer = utils.background({ command, stdio: [null, logs, logs] }); - - if (disconnect) { - zkSyncServer.unref(); - } - - // Server may need some time to recompile if it's a cold run, so wait for it. - let iter = 0; - let mainContract; - while (iter < 30 && !mainContract) { - try { - mainContract = await provider.getMainContractAddress(); - } catch (_) { - await utils.sleep(2); - iter += 1; - } - } - if (!mainContract) { - throw new Error('Server did not start'); - } - - await utils.sleep(10); -} From ffda3f79a99c5a1a0836a28f559cf03645156b4c Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Thu, 19 Sep 2024 16:48:59 +0200 Subject: [PATCH 20/51] style: format --- core/tests/ts-integration/src/utils.ts | 4 +-- core/tests/ts-integration/tests/fees.test.ts | 30 ++++++++++---------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index bc4830bb3fcf..22e35163c240 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -108,7 +108,7 @@ export class Node { public readonly tester: Tester, private readonly proc: ChildProcessWithoutNullStreams, private readonly type: TYPE - ) { } + ) {} public async terminate() { try { @@ -165,7 +165,7 @@ export class NodeSpawner { private readonly fileConfig: FileConfig, private readonly options: MainNodeSpawnOptions, private readonly env?: ProcessEnvOptions['env'] - ) { } + ) {} public async spawnMainNode(newL1GasPrice?: string, newPubdataPrice?: string): Promise> { const env = this.env ?? process.env; diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 80122b07ba16..68a747a480c5 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -29,22 +29,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; From 6b7233bada6e61cf537a636d5b187e5c2264730b Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 10:25:46 +0200 Subject: [PATCH 21/51] fix: update config on the fly --- core/tests/ts-integration/src/utils.ts | 56 +++++++++++++------- core/tests/ts-integration/tests/fees.test.ts | 6 +++ core/tests/ts-integration/tests/utils.ts | 29 ++++++++++ 3 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 core/tests/ts-integration/tests/utils.ts diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index 22e35163c240..aaa827631f62 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -6,6 +6,11 @@ import { killPidWithAllChilds } from 'utils/build/kill'; import * as utils from 'utils'; import fs from 'node:fs/promises'; import * as zksync from 'zksync-ethers'; +import { + setInternalL1PricingMultiplier, + setInternalPubdataPricingMultiplier, + setTransactionSlots +} from '../tests/utils'; // executes a command in background and returns a child process handle // by default pipes data to parent's stdio but this can be overridden @@ -50,8 +55,6 @@ export function runServerInBackground({ cwd, env, useZkInception, - newL1GasPrice, - newPubdataPrice, chain }: { components?: string[]; @@ -71,22 +74,6 @@ export function runServerInBackground({ } } else { command = 'zk server'; - command = `DATABASE_MERKLE_TREE_MODE=full ${command}`; - - if (newPubdataPrice) { - command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_PUBDATA_PRICE=${newPubdataPrice} ${command}`; - } - - if (newL1GasPrice) { - // We need to ensure that each transaction gets into its own batch for more fair comparison. - command = `ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_L1_GAS_PRICE=${newL1GasPrice} ${command}`; - } - - const testMode = newPubdataPrice || newL1GasPrice; - if (testMode) { - // We need to ensure that each transaction gets into its own batch for more fair comparison. - command = `CHAIN_STATE_KEEPER_TRANSACTION_SLOTS=1 ${command}`; - } } return runInBackground({ command, components, stdio, cwd, env }); } @@ -171,6 +158,37 @@ export class NodeSpawner { const env = this.env ?? process.env; const { fileConfig, pathToHome, options, logs } = this; + const testMode = newPubdataPrice || newL1GasPrice; + + console.log('New L1 Gas Price: ', newL1GasPrice); + console.log('New Pubdata Price: ', newPubdataPrice); + + if (fileConfig.loadFromFile) { + setTransactionSlots(pathToHome, fileConfig, testMode ? 1 : 8192); + setInternalL1PricingMultiplier(pathToHome, fileConfig, newL1GasPrice ? parseFloat(newL1GasPrice) : 0.8); + setInternalPubdataPricingMultiplier( + pathToHome, + fileConfig, + newPubdataPrice ? parseFloat(newPubdataPrice) : 1.0 + ); + } else { + env['DATABASE_MERKLE_TREE_MODE'] = 'full'; + + if (newPubdataPrice) { + env['ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_PUBDATA_PRICE'] = newPubdataPrice; + } + + if (newL1GasPrice) { + // We need to ensure that each transaction gets into its own batch for more fair comparison. + env['ETH_SENDER_GAS_ADJUSTER_INTERNAL_ENFORCED_L1_GAS_PRICE'] = newL1GasPrice; + } + + if (testMode) { + // We need to ensure that each transaction gets into its own batch for more fair comparison. + env['CHAIN_STATE_KEEPER_TRANSACTION_SLOTS'] = '1'; + } + } + let components = 'api,tree,eth,state_keeper,da_dispatcher,vm_runner_protective_reads'; if (options.enableConsensus) { components += ',consensus'; @@ -184,8 +202,6 @@ export class NodeSpawner { cwd: pathToHome, env: env, useZkInception: fileConfig.loadFromFile, - newL1GasPrice: newL1GasPrice, - newPubdataPrice: newPubdataPrice, chain: fileConfig.chain }); diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 68a747a480c5..420faa8a07df 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -21,6 +21,7 @@ import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { logsTestPath } from 'utils/build/logs'; import path from 'path'; import { NodeSpawner, Node, NodeType } from '../src/utils'; +import { setInternalL1PricingMultiplier, setInternalPubdataPricingMultiplier, setTransactionSlots } from './utils'; const UINT32_MAX = 2n ** 32n - 1n; const MAX_GAS_PER_PUBDATA = 50_000n; @@ -274,6 +275,11 @@ testFees('Test fees', function () { await mainNode.killAndWaitForShutdown(); mainNode = await mainNodeSpawner.spawnMainNode(); + // Restore defaults + setTransactionSlots(pathToHome, fileConfig, 8192); + setInternalL1PricingMultiplier(pathToHome, fileConfig, 0.8); + setInternalPubdataPricingMultiplier(pathToHome, fileConfig, 1.0); + await testMaster.deinitialize(); }); }); diff --git a/core/tests/ts-integration/tests/utils.ts b/core/tests/ts-integration/tests/utils.ts new file mode 100644 index 000000000000..9f46ee2c814e --- /dev/null +++ b/core/tests/ts-integration/tests/utils.ts @@ -0,0 +1,29 @@ +import * as fs from 'fs'; +import { getConfigPath } from 'utils/build/file-configs'; + +export function setInternalPubdataPricingMultiplier(pathToHome: string, fileConfig: any, value: number) { + setPropertyInGeneralConfig(pathToHome, fileConfig, 'internal_pubdata_pricing_multiplier', value); +} + +export function setInternalL1PricingMultiplier(pathToHome: string, fileConfig: any, value: number) { + setPropertyInGeneralConfig(pathToHome, fileConfig, 'internal_l1_pricing_multiplier', value); +} + +export function setTransactionSlots(pathToHome: string, fileConfig: any, value: number) { + setPropertyInGeneralConfig(pathToHome, fileConfig, 'transaction_slots', value); +} + +function setPropertyInGeneralConfig(pathToHome: string, fileConfig: any, property: string, value: number) { + const generalConfigPath = getConfigPath({ + pathToHome, + chain: fileConfig.chain, + configsFolder: 'configs', + config: 'general.yaml' + }); + const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); + + const regex = new RegExp(`${property}:\\s*\\d+(\\.\\d+)?`, 'g'); + const newGeneralConfig = generalConfig.replace(regex, `${property}: ${value}`); + + fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); +} From 7ab0b1c553eab72a16b769a79975b3647ee04bb9 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 10:42:54 +0200 Subject: [PATCH 22/51] feat: add L2 node PID to TestEnvironment --- core/tests/ts-integration/src/env.ts | 4 ++++ .../src/jest-setup/global-setup.ts | 20 +++---------------- core/tests/ts-integration/src/types.ts | 4 ++++ core/tests/ts-integration/tests/fees.test.ts | 15 ++++++++++++++ 4 files changed, 26 insertions(+), 17 deletions(-) diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index ffef0fce5ce3..1a58d55d4775 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -129,6 +129,7 @@ async function loadTestEnvironmentFromFile(chain: string): Promise { ); const healthcheckPort = process.env.API_HEALTHCHECK_PORT ?? '3071'; + const l2NodePid = 0; return { maxLogsLimit, pathToHome, @@ -257,6 +260,7 @@ export async function loadTestEnvironmentFromEnv(): Promise { network, mainWalletPK, l2NodeUrl, + l2NodePid, l1NodeUrl, wsL2NodeUrl, healthcheckPort, diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index 3b343ed91276..930a0d1509e5 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -3,8 +3,7 @@ import { bigIntReplacer } from '../helpers'; import { TestContextOwner, loadTestEnvironment } from '../index'; import path from 'path'; import { runServerInBackground } from '../utils'; -import utils, { sleep } from 'utils'; -import { killPidWithAllChilds } from 'utils/build/kill'; +import utils from 'utils'; declare global { var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; @@ -48,24 +47,11 @@ async function performSetup(_globalConfig: any, _projectConfig: any) { chain: fileConfig.chain }); - const testEnvironment = await loadTestEnvironment(); + let testEnvironment = await loadTestEnvironment(); + testEnvironment.l2NodePid = proc.pid!; const testContextOwner = new TestContextOwner(testEnvironment); const testContext = await testContextOwner.setupContext(); - // TODO: add PID to context and let the test suites kill the server. - try { - await killPidWithAllChilds(proc.pid!, 9); - } catch (err) { - console.log(`ignored error: ${err}`); - } - - let iter = 0; - while (iter < 30) { - console.log('Waiting for the server to stop...'); - await sleep(1); - iter += 1; - } - // Set the test context for test suites to pick up. // Currently, jest doesn't provide a way to pass data from `globalSetup` to suites, // so we store the data as serialized JSON. diff --git a/core/tests/ts-integration/src/types.ts b/core/tests/ts-integration/src/types.ts index 4975b7b612cf..222f39d36bf9 100644 --- a/core/tests/ts-integration/src/types.ts +++ b/core/tests/ts-integration/src/types.ts @@ -55,6 +55,10 @@ export interface TestEnvironment { * Mode of the l2 node */ nodeMode: NodeMode; + /* + * L2 node PID + */ + l2NodePid: number; /** * Plaintext name of the L1 network name (i.e. `localhost` or `goerli`). */ diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 420faa8a07df..0b1036cb382a 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -22,6 +22,7 @@ import { logsTestPath } from 'utils/build/logs'; import path from 'path'; import { NodeSpawner, Node, NodeType } from '../src/utils'; import { setInternalL1PricingMultiplier, setInternalPubdataPricingMultiplier, setTransactionSlots } from './utils'; +import { killPidWithAllChilds } from 'utils/build/kill'; const UINT32_MAX = 2n ** 32n - 1n; const MAX_GAS_PER_PUBDATA = 50_000n; @@ -76,6 +77,20 @@ testFees('Test fees', function () { testMaster = TestMaster.getInstance(__filename); alice = testMaster.mainAccount(); + try { + await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); + } catch (err) { + console.log(`ignored error: ${err}`); + } + + // TODO: imporove the waiting logic + let iter = 0; + while (iter < 10) { + console.log('Waiting for the server to stop...'); + await utils.sleep(1); + iter += 1; + } + tokenDetails = testMaster.environment().erc20Token; aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); From 7f62481d722172ef3517aeac38917aac7a6739d3 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 12:20:12 +0200 Subject: [PATCH 23/51] fix: set right properties on the fly --- core/tests/ts-integration/src/utils.ts | 24 +++++--- core/tests/ts-integration/tests/fees.test.ts | 6 +- core/tests/ts-integration/tests/utils.ts | 60 ++++++++++++++++++-- 3 files changed, 75 insertions(+), 15 deletions(-) diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index aaa827631f62..c04d0109e08e 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -7,8 +7,10 @@ import * as utils from 'utils'; import fs from 'node:fs/promises'; import * as zksync from 'zksync-ethers'; import { - setInternalL1PricingMultiplier, - setInternalPubdataPricingMultiplier, + deleteInternalEnforcedL1GasPrice, + deleteInternalEnforcedPubdataPrice, + setInternalEnforcedL1GasPrice, + setInternalEnforcedPubdataPrice, setTransactionSlots } from '../tests/utils'; @@ -165,12 +167,18 @@ export class NodeSpawner { if (fileConfig.loadFromFile) { setTransactionSlots(pathToHome, fileConfig, testMode ? 1 : 8192); - setInternalL1PricingMultiplier(pathToHome, fileConfig, newL1GasPrice ? parseFloat(newL1GasPrice) : 0.8); - setInternalPubdataPricingMultiplier( - pathToHome, - fileConfig, - newPubdataPrice ? parseFloat(newPubdataPrice) : 1.0 - ); + + if (newL1GasPrice) { + setInternalEnforcedL1GasPrice(pathToHome, fileConfig, parseFloat(newL1GasPrice)); + } else { + deleteInternalEnforcedL1GasPrice(pathToHome, fileConfig); + } + + if (newPubdataPrice) { + setInternalEnforcedPubdataPrice(pathToHome, fileConfig, parseFloat(newPubdataPrice)); + } else { + deleteInternalEnforcedPubdataPrice(pathToHome, fileConfig); + } } else { env['DATABASE_MERKLE_TREE_MODE'] = 'full'; diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 0b1036cb382a..952a46e3e308 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -21,7 +21,7 @@ import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { logsTestPath } from 'utils/build/logs'; import path from 'path'; import { NodeSpawner, Node, NodeType } from '../src/utils'; -import { setInternalL1PricingMultiplier, setInternalPubdataPricingMultiplier, setTransactionSlots } from './utils'; +import { deleteInternalEnforcedL1GasPrice, deleteInternalEnforcedPubdataPrice, setTransactionSlots } from './utils'; import { killPidWithAllChilds } from 'utils/build/kill'; const UINT32_MAX = 2n ** 32n - 1n; @@ -292,8 +292,8 @@ testFees('Test fees', function () { // Restore defaults setTransactionSlots(pathToHome, fileConfig, 8192); - setInternalL1PricingMultiplier(pathToHome, fileConfig, 0.8); - setInternalPubdataPricingMultiplier(pathToHome, fileConfig, 1.0); + deleteInternalEnforcedL1GasPrice(pathToHome, fileConfig); + deleteInternalEnforcedPubdataPrice(pathToHome, fileConfig); await testMaster.deinitialize(); }); diff --git a/core/tests/ts-integration/tests/utils.ts b/core/tests/ts-integration/tests/utils.ts index 9f46ee2c814e..72669cdf4555 100644 --- a/core/tests/ts-integration/tests/utils.ts +++ b/core/tests/ts-integration/tests/utils.ts @@ -1,12 +1,20 @@ import * as fs from 'fs'; import { getConfigPath } from 'utils/build/file-configs'; -export function setInternalPubdataPricingMultiplier(pathToHome: string, fileConfig: any, value: number) { - setPropertyInGeneralConfig(pathToHome, fileConfig, 'internal_pubdata_pricing_multiplier', value); +export function setInternalEnforcedPubdataPrice(pathToHome: string, fileConfig: any, value: number) { + setGasAdjusterProperty(pathToHome, fileConfig, 'internal_enforced_pubdata_price', value); } -export function setInternalL1PricingMultiplier(pathToHome: string, fileConfig: any, value: number) { - setPropertyInGeneralConfig(pathToHome, fileConfig, 'internal_l1_pricing_multiplier', value); +export function setInternalEnforcedL1GasPrice(pathToHome: string, fileConfig: any, value: number) { + setGasAdjusterProperty(pathToHome, fileConfig, 'internal_enforced_l1_gas_price', value); +} + +export function deleteInternalEnforcedPubdataPrice(pathToHome: string, fileConfig: any) { + deleteProperty(pathToHome, fileConfig, 'internal_enforced_pubdata_price'); +} + +export function deleteInternalEnforcedL1GasPrice(pathToHome: string, fileConfig: any) { + deleteProperty(pathToHome, fileConfig, 'internal_enforced_l1_gas_price'); } export function setTransactionSlots(pathToHome: string, fileConfig: any, value: number) { @@ -27,3 +35,47 @@ function setPropertyInGeneralConfig(pathToHome: string, fileConfig: any, propert fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); } + +function setGasAdjusterProperty(pathToHome: string, fileConfig: any, property: string, value: number) { + const generalConfigPath = getConfigPath({ + pathToHome, + chain: fileConfig.chain, + configsFolder: 'configs', + config: 'general.yaml' + }); + const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); + + // Define the regex pattern to check if the property already exists + const propertyRegex = new RegExp(`(^\\s*${property}:\\s*\\d+(\\.\\d+)?$)`, 'm'); + const gasAdjusterRegex = new RegExp('(^\\s*gas_adjuster:.*$)', 'gm'); + + let newGeneralConfig; + + if (propertyRegex.test(generalConfig)) { + // If the property exists, modify its value + newGeneralConfig = generalConfig.replace(propertyRegex, `${property}: ${value}`); + } else { + // If the property does not exist, add it under the gas_adjuster section + newGeneralConfig = generalConfig.replace(gasAdjusterRegex, `$1\n ${property}: ${value}`); + } + + fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); +} + +function deleteProperty(pathToHome: string, fileConfig: any, property: string) { + const generalConfigPath = getConfigPath({ + pathToHome, + chain: fileConfig.chain, + configsFolder: 'configs', + config: 'general.yaml' + }); + const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); + + // Define the regex pattern to find the property line and remove it completely + const propertyRegex = new RegExp(`^\\s*${property}:.*\\n?`, 'm'); + + // Remove the line if the property exists + const newGeneralConfig = generalConfig.replace(propertyRegex, ''); + + fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); +} From 539517b44b53904b8f4e2ac33ea6585f5f1b874d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 14:44:34 +0200 Subject: [PATCH 24/51] style: lint --- core/tests/ts-integration/src/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index c04d0109e08e..5ddd5061d3b2 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -1,4 +1,4 @@ -import { exec as _exec, spawn as _spawn, ChildProcessWithoutNullStreams, type ProcessEnvOptions } from 'child_process'; +import { spawn as _spawn, ChildProcessWithoutNullStreams, type ProcessEnvOptions } from 'child_process'; import { assert } from 'chai'; import { FileConfig } from 'utils/build/file-configs'; import { Tester } from './tester'; From eacd4decaa62b73d85087ef7d7fddad463f17cc6 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 14:45:47 +0200 Subject: [PATCH 25/51] fix: kill server before running fees.test.ts --- .github/workflows/ci-core-reusable.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index bb05a3318335..9ee6f77b1223 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -449,7 +449,9 @@ jobs: - name: Fee projection tests run: | - ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log + ci_run killall -INT zksync_server || true + + ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log ci_run zk_supervisor test fees --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log ci_run zk_supervisor test fees --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log ci_run zk_supervisor test fees --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log From b36f2d1c08309ccaeea01ab61e145f8ff7c75b2d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 14:51:53 +0200 Subject: [PATCH 26/51] fix: logs folder --- core/tests/ts-integration/tests/fees.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 952a46e3e308..085fb9a6e1e6 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -70,7 +70,7 @@ testFees('Test fees', function () { const enableConsensus = process.env.ENABLE_CONSENSUS == 'true'; async function logsPath(name: string): Promise { - return await logsTestPath(fileConfig.chain, 'logs/revert/', name); + return await logsTestPath(fileConfig.chain, 'logs/server/fees', name); } beforeAll(async () => { From e3e33ff80bb942d1b9669cbc18fd65bac27a711b Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 15:51:28 +0200 Subject: [PATCH 27/51] test: temporarily run fees tests only on era chain --- .github/workflows/ci-core-reusable.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index c4d996387142..a7f3472ba28b 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -462,9 +462,6 @@ jobs: ci_run killall -INT zksync_server || true ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log - ci_run zk_supervisor test fees --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log - ci_run zk_supervisor test fees --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log - ci_run zk_supervisor test fees --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log - name: Run revert tests run: | From 7e3889b6a5090f5c1348a20dabc235de0c9a8387 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:01:03 +0200 Subject: [PATCH 28/51] style: use const case for env variable in CI --- .github/workflows/ci-core-reusable.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index a7f3472ba28b..a816328eb56b 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -223,9 +223,9 @@ jobs: - name: Read Custom Token address and set as environment variable run: | - address=$(awk -F": " '/tokens:/ {found_tokens=1} found_tokens && /DAI:/ {found_dai=1} found_dai && /address:/ {print $2; exit}' ./configs/erc20.yaml) - echo "address=$address" - echo "address=$address" >> $GITHUB_ENV + CUSTOM_TOKEN_ADDRESS=$(awk -F": " '/tokens:/ {found_tokens=1} found_tokens && /DAI:/ {found_dai=1} found_dai && /address:/ {print $2; exit}' ./configs/erc20.yaml) + echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS" + echo "CUSTOM_TOKEN_ADDRESS=$CUSTOM_TOKEN_ADDRESS" >> $GITHUB_ENV - name: Create and initialize Validium chain run: | @@ -259,7 +259,7 @@ jobs: --prover-mode no-proofs \ --wallet-creation localhost \ --l1-batch-commit-data-generator-mode rollup \ - --base-token-address ${{ env.address }} \ + --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \ --base-token-price-nominator 3 \ --base-token-price-denominator 2 \ --set-as-default false \ @@ -318,7 +318,7 @@ jobs: --prover-mode no-proofs \ --wallet-creation localhost \ --l1-batch-commit-data-generator-mode validium \ - --base-token-address ${{ env.address }} \ + --base-token-address ${{ env.CUSTOM_TOKEN_ADDRESS }} \ --base-token-price-nominator 3 \ --base-token-price-denominator 2 \ --set-as-default false \ From 87a3d263821286e98b5448a21d46ff0ec9fa3049 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:02:35 +0200 Subject: [PATCH 29/51] style: format code --- .../crates/lib/prover_dal/src/cli_test_dal.rs | 99 ++++++++++--------- .../lib/prover_dal/src/fri_prover_dal.rs | 10 +- .../src/fri_witness_generator_dal.rs | 15 ++- 3 files changed, 71 insertions(+), 53 deletions(-) diff --git a/prover/crates/lib/prover_dal/src/cli_test_dal.rs b/prover/crates/lib/prover_dal/src/cli_test_dal.rs index 19fe0e4f57b0..d08418203378 100644 --- a/prover/crates/lib/prover_dal/src/cli_test_dal.rs +++ b/prover/crates/lib/prover_dal/src/cli_test_dal.rs @@ -21,11 +21,16 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - "UPDATE prover_jobs_fri SET status = $1 - WHERE l1_batch_number = $2 + r#" + UPDATE prover_jobs_fri + SET + status = $1 + WHERE + l1_batch_number = $2 AND sequence_number = $3 AND aggregation_round = $4 - AND circuit_id = $5", + AND circuit_id = $5 + "#, status.to_string(), batch_number.0 as i64, sequence_number as i64, @@ -44,7 +49,7 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - " + r#" INSERT INTO leaf_aggregation_witness_jobs_fri ( l1_batch_number, @@ -58,8 +63,9 @@ impl CliTestDal<'_, '_> { ($1, $2, 'waiting_for_proofs', 2, NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id) DO UPDATE - SET status = $3 - ", + SET + status = $3 + "#, batch_number.0 as i64, circuit_id as i16, status.to_string() @@ -76,21 +82,16 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - " + r#" INSERT INTO - node_aggregation_witness_jobs_fri ( - l1_batch_number, - circuit_id, - status, - created_at, - updated_at - ) + node_aggregation_witness_jobs_fri (l1_batch_number, circuit_id, status, created_at, updated_at) VALUES ($1, $2, 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id, depth) DO UPDATE - SET status = $3 - ", + SET + status = $3 + "#, batch_number.0 as i64, circuit_id as i16, status.to_string(), @@ -102,21 +103,16 @@ impl CliTestDal<'_, '_> { pub async fn insert_rt_job(&mut self, status: WitnessJobStatus, batch_number: L1BatchNumber) { sqlx::query!( - " + r#" INSERT INTO - recursion_tip_witness_jobs_fri ( - l1_batch_number, - status, - number_of_final_node_jobs, - created_at, - updated_at - ) + recursion_tip_witness_jobs_fri (l1_batch_number, status, number_of_final_node_jobs, created_at, updated_at) VALUES - ($1, 'waiting_for_proofs',1, NOW(), NOW()) + ($1, 'waiting_for_proofs', 1, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -131,7 +127,7 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - " + r#" INSERT INTO scheduler_witness_jobs_fri ( l1_batch_number, @@ -144,8 +140,9 @@ impl CliTestDal<'_, '_> { ($1, '', 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -160,20 +157,16 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - " + r#" INSERT INTO - proof_compression_jobs_fri ( - l1_batch_number, - status, - created_at, - updated_at - ) + proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at) VALUES ($1, $2, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -192,12 +185,17 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - "UPDATE prover_jobs_fri - SET status = $1, attempts = $2 - WHERE l1_batch_number = $3 - AND sequence_number =$4 + r#" + UPDATE prover_jobs_fri + SET + status = $1, + attempts = $2 + WHERE + l1_batch_number = $3 + AND sequence_number = $4 AND aggregation_round = $5 - AND circuit_id = $6", + AND circuit_id = $6 + "#, status.to_string(), attempts as i64, batch_number.0 as i64, @@ -218,10 +216,15 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - "UPDATE leaf_aggregation_witness_jobs_fri - SET status = $1, attempts = $2 - WHERE l1_batch_number = $3 - AND circuit_id = $4", + r#" + UPDATE leaf_aggregation_witness_jobs_fri + SET + status = $1, + attempts = $2 + WHERE + l1_batch_number = $3 + AND circuit_id = $4 + "#, status.to_string(), attempts as i64, batch_number.0 as i64, diff --git a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs index 1a3b8de0ce4b..71d0c11728b1 100644 --- a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs @@ -528,8 +528,14 @@ impl FriProverDal<'_, '_> { SELECT protocol_version AS "protocol_version!", protocol_version_patch AS "protocol_version_patch!", - COUNT(*) FILTER (WHERE status = 'queued') as queued, - COUNT(*) FILTER (WHERE status = 'in_progress') as in_progress + COUNT(*) FILTER ( + WHERE + status = 'queued' + ) AS queued, + COUNT(*) FILTER ( + WHERE + status = 'in_progress' + ) AS in_progress FROM prover_jobs_fri WHERE diff --git a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs index 66e34f7f8e75..c7ba0f60ef3f 100644 --- a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs @@ -1719,7 +1719,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, @@ -1786,7 +1789,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, @@ -1827,7 +1833,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, From 1699dd4df00a8b3f5a3d636332a111ac672f0633 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:09:32 +0200 Subject: [PATCH 30/51] fix: bring forward server start --- core/tests/ts-integration/tests/fees.test.ts | 39 ++++++++++---------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 085fb9a6e1e6..52813a1734d3 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; @@ -74,9 +74,6 @@ testFees('Test fees', function () { } beforeAll(async () => { - testMaster = TestMaster.getInstance(__filename); - alice = testMaster.mainAccount(); - try { await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); } catch (err) { @@ -91,6 +88,10 @@ testFees('Test fees', function () { iter += 1; } + mainNode = await mainNodeSpawner.spawnMainNode(); + + testMaster = TestMaster.getInstance(__filename); + alice = testMaster.mainAccount(); tokenDetails = testMaster.environment().erc20Token; aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); @@ -133,8 +134,6 @@ testFees('Test fees', function () { }); test('Test fees', async () => { - mainNode = await mainNodeSpawner.spawnMainNode(); - const receiver = ethers.Wallet.createRandom().address; // Getting ETH price in gas. From 2ffcd926905c14e61c223e942984e8ed6c70dcc2 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:10:01 +0200 Subject: [PATCH 31/51] style: format --- core/tests/ts-integration/tests/fees.test.ts | 30 ++++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 52813a1734d3..0045254fca28 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; From 5ba3f36c253ef9855a2140a12d935b8502e7340d Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:16:51 +0200 Subject: [PATCH 32/51] fix: read test environment before killing the node --- core/tests/ts-integration/tests/fees.test.ts | 33 ++++++++++---------- yarn.lock | 2 +- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 0045254fca28..64b2646e5f22 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; @@ -74,6 +74,8 @@ testFees('Test fees', function () { } beforeAll(async () => { + testMaster = TestMaster.getInstance(__filename); + try { await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); } catch (err) { @@ -90,7 +92,6 @@ testFees('Test fees', function () { mainNode = await mainNodeSpawner.spawnMainNode(); - testMaster = TestMaster.getInstance(__filename); alice = testMaster.mainAccount(); tokenDetails = testMaster.environment().erc20Token; aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); diff --git a/yarn.lock b/yarn.lock index b70e64f148a1..3c764c7c7b7f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6903,7 +6903,7 @@ jest-each@^29.7.0: jest-util "^29.7.0" pretty-format "^29.7.0" -jest-environment-node@^29.7.0: +jest-environment-node@^29.0.3, jest-environment-node@^29.7.0: version "29.7.0" resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.7.0.tgz#0b93e111dda8ec120bc8300e6d1fb9576e164376" integrity sha512-DOSwCRqXirTOyheM+4d5YZOrWcdu0LNZ87ewUoywbcb2XR4wKgqiG8vNeYwhjFMbEkfju7wx2GYH0P2gevGvFw== From a12bf21511d85054fb39ee6a0a0ee0fe2c9e4e2f Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:18:50 +0200 Subject: [PATCH 33/51] fix: fix operations in beforeAll step --- core/tests/ts-integration/tests/fees.test.ts | 42 ++++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 64b2646e5f22..1755f905994c 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; @@ -90,12 +90,6 @@ testFees('Test fees', function () { iter += 1; } - mainNode = await mainNodeSpawner.spawnMainNode(); - - alice = testMaster.mainAccount(); - tokenDetails = testMaster.environment().erc20Token; - aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); - if (!fileConfig.loadFromFile) { ethClientWeb3Url = process.env.ETH_CLIENT_WEB3_URL!; apiWeb3JsonRpcHttpUrl = process.env.API_WEB3_JSON_RPC_HTTP_URL!; @@ -132,6 +126,12 @@ testFees('Test fees', function () { apiWeb3JsonRpcHttpUrl, baseTokenAddress }); + + mainNode = await mainNodeSpawner.spawnMainNode(); + + alice = testMaster.mainAccount(); + tokenDetails = testMaster.environment().erc20Token; + aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); }); test('Test fees', async () => { From 285289dae9cb4319c10703a4abb12108ac2615d7 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 16:40:06 +0200 Subject: [PATCH 34/51] Revert "style: format code" This reverts commit 87a3d263821286e98b5448a21d46ff0ec9fa3049. --- .../crates/lib/prover_dal/src/cli_test_dal.rs | 99 +++++++++---------- .../lib/prover_dal/src/fri_prover_dal.rs | 10 +- .../src/fri_witness_generator_dal.rs | 15 +-- 3 files changed, 53 insertions(+), 71 deletions(-) diff --git a/prover/crates/lib/prover_dal/src/cli_test_dal.rs b/prover/crates/lib/prover_dal/src/cli_test_dal.rs index d08418203378..19fe0e4f57b0 100644 --- a/prover/crates/lib/prover_dal/src/cli_test_dal.rs +++ b/prover/crates/lib/prover_dal/src/cli_test_dal.rs @@ -21,16 +21,11 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - r#" - UPDATE prover_jobs_fri - SET - status = $1 - WHERE - l1_batch_number = $2 + "UPDATE prover_jobs_fri SET status = $1 + WHERE l1_batch_number = $2 AND sequence_number = $3 AND aggregation_round = $4 - AND circuit_id = $5 - "#, + AND circuit_id = $5", status.to_string(), batch_number.0 as i64, sequence_number as i64, @@ -49,7 +44,7 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - r#" + " INSERT INTO leaf_aggregation_witness_jobs_fri ( l1_batch_number, @@ -63,9 +58,8 @@ impl CliTestDal<'_, '_> { ($1, $2, 'waiting_for_proofs', 2, NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id) DO UPDATE - SET - status = $3 - "#, + SET status = $3 + ", batch_number.0 as i64, circuit_id as i16, status.to_string() @@ -82,16 +76,21 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - r#" + " INSERT INTO - node_aggregation_witness_jobs_fri (l1_batch_number, circuit_id, status, created_at, updated_at) + node_aggregation_witness_jobs_fri ( + l1_batch_number, + circuit_id, + status, + created_at, + updated_at + ) VALUES ($1, $2, 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id, depth) DO UPDATE - SET - status = $3 - "#, + SET status = $3 + ", batch_number.0 as i64, circuit_id as i16, status.to_string(), @@ -103,16 +102,21 @@ impl CliTestDal<'_, '_> { pub async fn insert_rt_job(&mut self, status: WitnessJobStatus, batch_number: L1BatchNumber) { sqlx::query!( - r#" + " INSERT INTO - recursion_tip_witness_jobs_fri (l1_batch_number, status, number_of_final_node_jobs, created_at, updated_at) + recursion_tip_witness_jobs_fri ( + l1_batch_number, + status, + number_of_final_node_jobs, + created_at, + updated_at + ) VALUES - ($1, 'waiting_for_proofs', 1, NOW(), NOW()) + ($1, 'waiting_for_proofs',1, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET - status = $2 - "#, + SET status = $2 + ", batch_number.0 as i64, status.to_string(), ) @@ -127,7 +131,7 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - r#" + " INSERT INTO scheduler_witness_jobs_fri ( l1_batch_number, @@ -140,9 +144,8 @@ impl CliTestDal<'_, '_> { ($1, '', 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET - status = $2 - "#, + SET status = $2 + ", batch_number.0 as i64, status.to_string(), ) @@ -157,16 +160,20 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - r#" + " INSERT INTO - proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at) + proof_compression_jobs_fri ( + l1_batch_number, + status, + created_at, + updated_at + ) VALUES ($1, $2, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET - status = $2 - "#, + SET status = $2 + ", batch_number.0 as i64, status.to_string(), ) @@ -185,17 +192,12 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - r#" - UPDATE prover_jobs_fri - SET - status = $1, - attempts = $2 - WHERE - l1_batch_number = $3 - AND sequence_number = $4 + "UPDATE prover_jobs_fri + SET status = $1, attempts = $2 + WHERE l1_batch_number = $3 + AND sequence_number =$4 AND aggregation_round = $5 - AND circuit_id = $6 - "#, + AND circuit_id = $6", status.to_string(), attempts as i64, batch_number.0 as i64, @@ -216,15 +218,10 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - r#" - UPDATE leaf_aggregation_witness_jobs_fri - SET - status = $1, - attempts = $2 - WHERE - l1_batch_number = $3 - AND circuit_id = $4 - "#, + "UPDATE leaf_aggregation_witness_jobs_fri + SET status = $1, attempts = $2 + WHERE l1_batch_number = $3 + AND circuit_id = $4", status.to_string(), attempts as i64, batch_number.0 as i64, diff --git a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs index 71d0c11728b1..1a3b8de0ce4b 100644 --- a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs @@ -528,14 +528,8 @@ impl FriProverDal<'_, '_> { SELECT protocol_version AS "protocol_version!", protocol_version_patch AS "protocol_version_patch!", - COUNT(*) FILTER ( - WHERE - status = 'queued' - ) AS queued, - COUNT(*) FILTER ( - WHERE - status = 'in_progress' - ) AS in_progress + COUNT(*) FILTER (WHERE status = 'queued') as queued, + COUNT(*) FILTER (WHERE status = 'in_progress') as in_progress FROM prover_jobs_fri WHERE diff --git a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs index c7ba0f60ef3f..66e34f7f8e75 100644 --- a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs @@ -1719,10 +1719,7 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND ( - status = 'in_progress' - OR status = 'failed' - ) + AND (status = 'in_progress' OR status = 'failed') RETURNING l1_batch_number, status, @@ -1789,10 +1786,7 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND ( - status = 'in_progress' - OR status = 'failed' - ) + AND (status = 'in_progress' OR status = 'failed') RETURNING l1_batch_number, status, @@ -1833,10 +1827,7 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND ( - status = 'in_progress' - OR status = 'failed' - ) + AND (status = 'in_progress' OR status = 'failed') RETURNING l1_batch_number, status, From fb75f048851e42d9d3bcb54c7625307efdbc4cdd Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 17:04:00 +0200 Subject: [PATCH 35/51] fix: remove extra node restart --- core/tests/ts-integration/tests/fees.test.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 1755f905994c..c04ea4f49f2b 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -220,9 +220,6 @@ testFees('Test fees', function () { }); test('Test gas consumption under large L1 gas price', async () => { - await mainNode.killAndWaitForShutdown(); - mainNode = await mainNodeSpawner.spawnMainNode(); - if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. return; From a936b0414111442f126e1847d321c60ff4ab5a1c Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Fri, 20 Sep 2024 18:33:35 +0200 Subject: [PATCH 36/51] refactor: remove unused code from tester --- core/tests/ts-integration/src/tester.ts | 52 +------------------------ 1 file changed, 1 insertion(+), 51 deletions(-) diff --git a/core/tests/ts-integration/src/tester.ts b/core/tests/ts-integration/src/tester.ts index 1809b4c2784c..cf2d84f95023 100644 --- a/core/tests/ts-integration/src/tester.ts +++ b/core/tests/ts-integration/src/tester.ts @@ -1,11 +1,8 @@ -import { expect } from 'chai'; import * as ethers from 'ethers'; import * as zksync from 'zksync-ethers'; import * as fs from 'fs'; import * as path from 'path'; -const BASE_ERC20_TO_MINT = ethers.parseEther('100'); - export class Tester { public runningFee: Map; @@ -26,7 +23,7 @@ export class Tester { ethProvider.pollingInterval = 100; const testConfigPath = path.join(process.env.ZKSYNC_HOME!, `etc/test_config/constant`); - const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, {encoding: 'utf-8'})); + const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, { encoding: 'utf-8' })); let ethWalletPK: string; if (process.env.MASTER_WALLET_PK) { @@ -75,51 +72,4 @@ export class Tester { return new Tester(ethProvider, ethWallet, syncWallet, web3Provider, isETHBasedChain, baseTokenAddress); } - - /// Ensures that the main wallet has enough base token. - /// This can not be done inside the `init` function because `init` function can be called before the - /// L2 RPC is active, but we need the L2 RPC to get the base token address. - async fundSyncWallet() { - const baseTokenAddress = await this.syncWallet.provider.getBaseTokenContractAddress(); - if (!(baseTokenAddress === zksync.utils.ETH_ADDRESS_IN_CONTRACTS)) { - const l1Erc20ABI = ['function mint(address to, uint256 amount)']; - const l1Erc20Contract = new ethers.Contract(baseTokenAddress, l1Erc20ABI, this.ethWallet); - await (await l1Erc20Contract.mint(this.ethWallet.address, BASE_ERC20_TO_MINT)).wait(); - } - } - - async fundedWallet(ethAmount: bigint, l1Token: zksync.types.Address, tokenAmount: bigint) { - const newWalletHD = zksync.Wallet.createRandom(); - const newWallet = new zksync.Wallet(newWalletHD.privateKey, this.web3Provider, this.ethProvider); - - let ethBalance = await this.syncWallet.getBalanceL1(); - expect(ethBalance > ethAmount, 'Insufficient eth balance to create funded wallet').to.be.true; - - // To make the wallet capable of requesting priority operations, - // send ETH to L1. - - const tx1 = await this.syncWallet.ethWallet().sendTransaction({ - to: newWallet.address, - value: ethAmount - }); - await tx1.wait(); - - // Funds the wallet with L1 token. - - let tokenBalance = await this.syncWallet.getBalanceL1(l1Token); - expect(tokenBalance > tokenAmount, 'Insufficient token balance to create funded wallet').to.be.true; - - const erc20ABI = ['function transfer(address to, uint256 amount)']; - const erc20Contract = new ethers.Contract(l1Token, erc20ABI, this.ethWallet); - - const tx2 = await erc20Contract.transfer(newWallet.address, tokenAmount); - await tx2.wait(); - - return newWallet; - } - - emptyWallet() { - const walletHD = zksync.Wallet.createRandom(); - return new zksync.Wallet(walletHD.privateKey, this.web3Provider, this.ethProvider); - } } From a5d2f8fa7bcec9daf114efe81f68c7bd59540a2f Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Sun, 22 Sep 2024 09:24:28 +0200 Subject: [PATCH 37/51] fix: unref node processes --- core/tests/ts-integration/src/jest-setup/global-setup.ts | 2 ++ core/tests/ts-integration/src/utils.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index 0480a99e91fd..37e6b69ec836 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -48,6 +48,8 @@ async function performSetup(_globalConfig: any, _projectConfig: any) { chain: fileConfig.chain }); + proc.unref(); + let testEnvironment = await loadTestEnvironment(); testEnvironment.l2NodePid = proc.pid!; const testContextOwner = new TestContextOwner(testEnvironment); diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index 5ddd5061d3b2..ef161400f2cd 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -212,6 +212,7 @@ export class NodeSpawner { useZkInception: fileConfig.loadFromFile, chain: fileConfig.chain }); + proc.unref(); // Wait until the main node starts responding. const tester = await Tester.init( From 19138a78626e4a072f516802566a88b13fff4369 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Sun, 22 Sep 2024 09:48:59 +0200 Subject: [PATCH 38/51] fix: unref only when needed --- core/tests/ts-integration/src/utils.ts | 3 +-- core/tests/ts-integration/tests/fees.test.ts | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index ef161400f2cd..627867c214c0 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -95,7 +95,7 @@ export enum NodeType { export class Node { constructor( public readonly tester: Tester, - private readonly proc: ChildProcessWithoutNullStreams, + public proc: ChildProcessWithoutNullStreams, private readonly type: TYPE ) {} @@ -212,7 +212,6 @@ export class NodeSpawner { useZkInception: fileConfig.loadFromFile, chain: fileConfig.chain }); - proc.unref(); // Wait until the main node starts responding. const tester = await Tester.init( diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index c04ea4f49f2b..b86ce2bcb594 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -286,6 +286,7 @@ testFees('Test fees', function () { // Returning the pubdata price to the default one await mainNode.killAndWaitForShutdown(); mainNode = await mainNodeSpawner.spawnMainNode(); + mainNode.proc.unref(); // Restore defaults setTransactionSlots(pathToHome, fileConfig, 8192); From 44f743b69fa10dbf2b7281dfce3652fbad502494 Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Sun, 22 Sep 2024 10:50:00 +0200 Subject: [PATCH 39/51] refactor: simplify waiting --- core/tests/ts-integration/tests/fees.test.ts | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index b86ce2bcb594..4a8d778d4429 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -83,12 +83,8 @@ testFees('Test fees', function () { } // TODO: imporove the waiting logic - let iter = 0; - while (iter < 10) { - console.log('Waiting for the server to stop...'); - await utils.sleep(1); - iter += 1; - } + console.log('Waiting for the server to stop...'); + await utils.sleep(10); if (!fileConfig.loadFromFile) { ethClientWeb3Url = process.env.ETH_CLIENT_WEB3_URL!; From 1ce733b309406e1b5a179208fe3ee5edc2334adc Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 24 Sep 2024 11:42:27 +0200 Subject: [PATCH 40/51] Fix queries Signed-off-by: Danil --- docs/guides/external-node/00_quick_start.md | 3 +- ...f1cdac8b194f09926c133985479c533a651f2.json | 18 ++++ ...0d001cdf5bc7ba988b742571ec90a938434e3.json | 17 ---- ...0cd5c6b1122c35d7ffdbbba30327ca3fb5a8.json} | 4 +- ...cf8c93630d529ec96e67aac078f18196f61a5.json | 19 ++++ ...adefab0bf3abf6853a6d36123c8adcaf813b.json} | 4 +- ...d48c95ca5b4520dde415a2b5ff32ece47c86.json} | 4 +- ...8d4cc59246dda91b19526e73f27a17c8e3da.json} | 4 +- ...a468057599be1e6c6c96a947c33df53a68224.json | 15 --- ...2e61157bf58aec70903623afc9da24d46a336.json | 16 --- ...1b931c0d8dbc6835dfac20107ea7412ce9fbb.json | 15 --- ...9a67936d572f8046d3a1c7a4f100ff209d81d.json | 18 ---- ...41976a264759c4060c1a38e466ee2052fc17d.json | 15 +++ ...2a4a98ec63eb942c73ce4448d0957346047cd.json | 17 ++++ ...08a01b63ae4aa03c983c3a52c802d585e5a80.json | 15 +++ ...700302981be0afef31a8864613484f8521f9e.json | 19 ---- ...b5a4672ad50a9de92c84d939ac4c69842e355.json | 16 +++ ...d005d8760c4809b7aef902155196873da66e.json} | 4 +- ...8e1010d7389457b3c97e9b238a3a0291a54e.json} | 4 +- .../crates/lib/prover_dal/src/cli_test_dal.rs | 99 ++++++++++--------- .../lib/prover_dal/src/fri_prover_dal.rs | 10 +- .../src/fri_witness_generator_dal.rs | 15 ++- 22 files changed, 184 insertions(+), 167 deletions(-) create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-1297f0977132185d6bd4501f490f1cdac8b194f09926c133985479c533a651f2.json delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-1926cf432237684de2383179a6d0d001cdf5bc7ba988b742571ec90a938434e3.json rename prover/crates/lib/prover_dal/.sqlx/{query-97adb49780c9edde6a3cfda09dadbd694e1781e013247d090a280a1f894de464.json => query-29f7a564a8373f7e44840e8e9e7d0cd5c6b1122c35d7ffdbbba30327ca3fb5a8.json} (53%) create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-2d1461e068e43fd16714610b383cf8c93630d529ec96e67aac078f18196f61a5.json rename prover/crates/lib/prover_dal/.sqlx/{query-75c1affbca0901edd5d0e2f12ef4d935674a5aff2f34421d753b4d1a9dea5b12.json => query-35a76415cb746d03da31481edc65adefab0bf3abf6853a6d36123c8adcaf813b.json} (70%) rename prover/crates/lib/prover_dal/.sqlx/{query-548414f8148740c991c345e5fd46ea738d209eb07e7a6bcbdb33e25b3347a08c.json => query-3727d5614d2fe2a4d96f880eb72cd48c95ca5b4520dde415a2b5ff32ece47c86.json} (70%) rename prover/crates/lib/prover_dal/.sqlx/{query-c19fc4c8e4b3a3ef4f9c0f4c22ed68c598eada8e60938a8e4b5cd32b53f5a574.json => query-37ad15f54f4a6f4f79c71a857f3a8d4cc59246dda91b19526e73f27a17c8e3da.json} (69%) delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-39f60c638d445c5dbf23e01fd89a468057599be1e6c6c96a947c33df53a68224.json delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-3a9ffd4d88f2cfac22835aac2512e61157bf58aec70903623afc9da24d46a336.json delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-3bb8fbd9e83703887e0a3c196031b931c0d8dbc6835dfac20107ea7412ce9fbb.json delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-434f7cb51a7d22948cd26e962679a67936d572f8046d3a1c7a4f100ff209d81d.json create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-73266a8526c6adc315900e2e95441976a264759c4060c1a38e466ee2052fc17d.json create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-9730c8225ff2cf3111185e81f602a4a98ec63eb942c73ce4448d0957346047cd.json create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-a817f0fec85388b3e2510ce259208a01b63ae4aa03c983c3a52c802d585e5a80.json delete mode 100644 prover/crates/lib/prover_dal/.sqlx/query-aabcfa9005b8e1d84cfa083a47a700302981be0afef31a8864613484f8521f9e.json create mode 100644 prover/crates/lib/prover_dal/.sqlx/query-c8daa62b3835c15fafb3f83deabb5a4672ad50a9de92c84d939ac4c69842e355.json rename prover/crates/lib/prover_dal/.sqlx/{query-63cf7038e6c48af8ed9afc7d6ea07edd87cb16a79c13e7d4291d99736e51d3b9.json => query-e875dcbbdaed6998dbea45d4eab5d005d8760c4809b7aef902155196873da66e.json} (82%) rename prover/crates/lib/prover_dal/.sqlx/{query-0eac6f7b2d799059328584029b437891598dc79b5ed11258b2c90c3f282929ad.json => query-eec29cbff034818f4fb5ec1e6ad38e1010d7389457b3c97e9b238a3a0291a54e.json} (78%) diff --git a/docs/guides/external-node/00_quick_start.md b/docs/guides/external-node/00_quick_start.md index 75d8ba891512..287a4d2d47c0 100644 --- a/docs/guides/external-node/00_quick_start.md +++ b/docs/guides/external-node/00_quick_start.md @@ -34,8 +34,7 @@ cd docker-compose-examples docker compose --file testnet-external-node-docker-compose.yml down --volumes ``` -You can see the status of the node (after recovery) in -[local grafana dashboard](http://localhost:3000/dashboards). +You can see the status of the node (after recovery) in [local grafana dashboard](http://localhost:3000/dashboards). Those commands start ZKsync node locally inside docker. diff --git a/prover/crates/lib/prover_dal/.sqlx/query-1297f0977132185d6bd4501f490f1cdac8b194f09926c133985479c533a651f2.json b/prover/crates/lib/prover_dal/.sqlx/query-1297f0977132185d6bd4501f490f1cdac8b194f09926c133985479c533a651f2.json new file mode 100644 index 000000000000..c99572bcc8e5 --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-1297f0977132185d6bd4501f490f1cdac8b194f09926c133985479c533a651f2.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE prover_jobs_fri\n SET\n status = $1\n WHERE\n l1_batch_number = $2\n AND sequence_number = $3\n AND aggregation_round = $4\n AND circuit_id = $5\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int8", + "Int4", + "Int2", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "1297f0977132185d6bd4501f490f1cdac8b194f09926c133985479c533a651f2" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-1926cf432237684de2383179a6d0d001cdf5bc7ba988b742571ec90a938434e3.json b/prover/crates/lib/prover_dal/.sqlx/query-1926cf432237684de2383179a6d0d001cdf5bc7ba988b742571ec90a938434e3.json deleted file mode 100644 index 4015a22ff3fd..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-1926cf432237684de2383179a6d0d001cdf5bc7ba988b742571ec90a938434e3.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE leaf_aggregation_witness_jobs_fri \n SET status = $1, attempts = $2\n WHERE l1_batch_number = $3\n AND circuit_id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int2", - "Int8", - "Int2" - ] - }, - "nullable": [] - }, - "hash": "1926cf432237684de2383179a6d0d001cdf5bc7ba988b742571ec90a938434e3" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-97adb49780c9edde6a3cfda09dadbd694e1781e013247d090a280a1f894de464.json b/prover/crates/lib/prover_dal/.sqlx/query-29f7a564a8373f7e44840e8e9e7d0cd5c6b1122c35d7ffdbbba30327ca3fb5a8.json similarity index 53% rename from prover/crates/lib/prover_dal/.sqlx/query-97adb49780c9edde6a3cfda09dadbd694e1781e013247d090a280a1f894de464.json rename to prover/crates/lib/prover_dal/.sqlx/query-29f7a564a8373f7e44840e8e9e7d0cd5c6b1122c35d7ffdbbba30327ca3fb5a8.json index ce9e492a7d4a..05163dcfa2e6 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-97adb49780c9edde6a3cfda09dadbd694e1781e013247d090a280a1f894de464.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-29f7a564a8373f7e44840e8e9e7d0cd5c6b1122c35d7ffdbbba30327ca3fb5a8.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n protocol_version AS \"protocol_version!\",\n protocol_version_patch AS \"protocol_version_patch!\",\n COUNT(*) FILTER (WHERE status = 'queued') as queued,\n COUNT(*) FILTER (WHERE status = 'in_progress') as in_progress\n FROM\n prover_jobs_fri\n WHERE\n status IN ('queued', 'in_progress')\n AND protocol_version IS NOT NULL\n GROUP BY\n protocol_version,\n protocol_version_patch\n ", + "query": "\n SELECT\n protocol_version AS \"protocol_version!\",\n protocol_version_patch AS \"protocol_version_patch!\",\n COUNT(*) FILTER (\n WHERE\n status = 'queued'\n ) AS queued,\n COUNT(*) FILTER (\n WHERE\n status = 'in_progress'\n ) AS in_progress\n FROM\n prover_jobs_fri\n WHERE\n status IN ('queued', 'in_progress')\n AND protocol_version IS NOT NULL\n GROUP BY\n protocol_version,\n protocol_version_patch\n ", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "97adb49780c9edde6a3cfda09dadbd694e1781e013247d090a280a1f894de464" + "hash": "29f7a564a8373f7e44840e8e9e7d0cd5c6b1122c35d7ffdbbba30327ca3fb5a8" } diff --git a/prover/crates/lib/prover_dal/.sqlx/query-2d1461e068e43fd16714610b383cf8c93630d529ec96e67aac078f18196f61a5.json b/prover/crates/lib/prover_dal/.sqlx/query-2d1461e068e43fd16714610b383cf8c93630d529ec96e67aac078f18196f61a5.json new file mode 100644 index 000000000000..50d121213fb9 --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-2d1461e068e43fd16714610b383cf8c93630d529ec96e67aac078f18196f61a5.json @@ -0,0 +1,19 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE prover_jobs_fri\n SET\n status = $1,\n attempts = $2\n WHERE\n l1_batch_number = $3\n AND sequence_number = $4\n AND aggregation_round = $5\n AND circuit_id = $6\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int2", + "Int8", + "Int4", + "Int2", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "2d1461e068e43fd16714610b383cf8c93630d529ec96e67aac078f18196f61a5" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-75c1affbca0901edd5d0e2f12ef4d935674a5aff2f34421d753b4d1a9dea5b12.json b/prover/crates/lib/prover_dal/.sqlx/query-35a76415cb746d03da31481edc65adefab0bf3abf6853a6d36123c8adcaf813b.json similarity index 70% rename from prover/crates/lib/prover_dal/.sqlx/query-75c1affbca0901edd5d0e2f12ef4d935674a5aff2f34421d753b4d1a9dea5b12.json rename to prover/crates/lib/prover_dal/.sqlx/query-35a76415cb746d03da31481edc65adefab0bf3abf6853a6d36123c8adcaf813b.json index 14463ecbe426..bf8db798e7d4 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-75c1affbca0901edd5d0e2f12ef4d935674a5aff2f34421d753b4d1a9dea5b12.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-35a76415cb746d03da31481edc65adefab0bf3abf6853a6d36123c8adcaf813b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE scheduler_witness_jobs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (status = 'in_progress' OR status = 'failed')\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", + "query": "\n UPDATE scheduler_witness_jobs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (\n status = 'in_progress'\n OR status = 'failed'\n )\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", "describe": { "columns": [ { @@ -43,5 +43,5 @@ true ] }, - "hash": "75c1affbca0901edd5d0e2f12ef4d935674a5aff2f34421d753b4d1a9dea5b12" + "hash": "35a76415cb746d03da31481edc65adefab0bf3abf6853a6d36123c8adcaf813b" } diff --git a/prover/crates/lib/prover_dal/.sqlx/query-548414f8148740c991c345e5fd46ea738d209eb07e7a6bcbdb33e25b3347a08c.json b/prover/crates/lib/prover_dal/.sqlx/query-3727d5614d2fe2a4d96f880eb72cd48c95ca5b4520dde415a2b5ff32ece47c86.json similarity index 70% rename from prover/crates/lib/prover_dal/.sqlx/query-548414f8148740c991c345e5fd46ea738d209eb07e7a6bcbdb33e25b3347a08c.json rename to prover/crates/lib/prover_dal/.sqlx/query-3727d5614d2fe2a4d96f880eb72cd48c95ca5b4520dde415a2b5ff32ece47c86.json index 8f5b046b974f..d7eb6a32b421 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-548414f8148740c991c345e5fd46ea738d209eb07e7a6bcbdb33e25b3347a08c.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-3727d5614d2fe2a4d96f880eb72cd48c95ca5b4520dde415a2b5ff32ece47c86.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE witness_inputs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (status = 'in_progress' OR status = 'failed')\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", + "query": "\n UPDATE witness_inputs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (\n status = 'in_progress'\n OR status = 'failed'\n )\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", "describe": { "columns": [ { @@ -43,5 +43,5 @@ true ] }, - "hash": "548414f8148740c991c345e5fd46ea738d209eb07e7a6bcbdb33e25b3347a08c" + "hash": "3727d5614d2fe2a4d96f880eb72cd48c95ca5b4520dde415a2b5ff32ece47c86" } diff --git a/prover/crates/lib/prover_dal/.sqlx/query-c19fc4c8e4b3a3ef4f9c0f4c22ed68c598eada8e60938a8e4b5cd32b53f5a574.json b/prover/crates/lib/prover_dal/.sqlx/query-37ad15f54f4a6f4f79c71a857f3a8d4cc59246dda91b19526e73f27a17c8e3da.json similarity index 69% rename from prover/crates/lib/prover_dal/.sqlx/query-c19fc4c8e4b3a3ef4f9c0f4c22ed68c598eada8e60938a8e4b5cd32b53f5a574.json rename to prover/crates/lib/prover_dal/.sqlx/query-37ad15f54f4a6f4f79c71a857f3a8d4cc59246dda91b19526e73f27a17c8e3da.json index 3c4c8d7a29f3..c97fe7f4042b 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-c19fc4c8e4b3a3ef4f9c0f4c22ed68c598eada8e60938a8e4b5cd32b53f5a574.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-37ad15f54f4a6f4f79c71a857f3a8d4cc59246dda91b19526e73f27a17c8e3da.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (status = 'in_progress' OR status = 'failed')\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", + "query": "\n UPDATE recursion_tip_witness_jobs_fri\n SET\n status = 'queued',\n updated_at = NOW(),\n processing_started_at = NOW()\n WHERE\n l1_batch_number = $1\n AND attempts >= $2\n AND (\n status = 'in_progress'\n OR status = 'failed'\n )\n RETURNING\n l1_batch_number,\n status,\n attempts,\n error,\n picked_by\n ", "describe": { "columns": [ { @@ -43,5 +43,5 @@ true ] }, - "hash": "c19fc4c8e4b3a3ef4f9c0f4c22ed68c598eada8e60938a8e4b5cd32b53f5a574" + "hash": "37ad15f54f4a6f4f79c71a857f3a8d4cc59246dda91b19526e73f27a17c8e3da" } diff --git a/prover/crates/lib/prover_dal/.sqlx/query-39f60c638d445c5dbf23e01fd89a468057599be1e6c6c96a947c33df53a68224.json b/prover/crates/lib/prover_dal/.sqlx/query-39f60c638d445c5dbf23e01fd89a468057599be1e6c6c96a947c33df53a68224.json deleted file mode 100644 index 5cec4d7d7d03..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-39f60c638d445c5dbf23e01fd89a468057599be1e6c6c96a947c33df53a68224.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n recursion_tip_witness_jobs_fri (\n l1_batch_number,\n status,\n number_of_final_node_jobs,\n created_at,\n updated_at\n )\n VALUES\n ($1, 'waiting_for_proofs',1, NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET status = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "39f60c638d445c5dbf23e01fd89a468057599be1e6c6c96a947c33df53a68224" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-3a9ffd4d88f2cfac22835aac2512e61157bf58aec70903623afc9da24d46a336.json b/prover/crates/lib/prover_dal/.sqlx/query-3a9ffd4d88f2cfac22835aac2512e61157bf58aec70903623afc9da24d46a336.json deleted file mode 100644 index 063ae8fc90a3..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-3a9ffd4d88f2cfac22835aac2512e61157bf58aec70903623afc9da24d46a336.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n node_aggregation_witness_jobs_fri (\n l1_batch_number,\n circuit_id,\n status,\n created_at,\n updated_at\n )\n VALUES\n ($1, $2, 'waiting_for_proofs', NOW(), NOW())\n ON CONFLICT (l1_batch_number, circuit_id, depth) DO\n UPDATE\n SET status = $3\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int2", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3a9ffd4d88f2cfac22835aac2512e61157bf58aec70903623afc9da24d46a336" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-3bb8fbd9e83703887e0a3c196031b931c0d8dbc6835dfac20107ea7412ce9fbb.json b/prover/crates/lib/prover_dal/.sqlx/query-3bb8fbd9e83703887e0a3c196031b931c0d8dbc6835dfac20107ea7412ce9fbb.json deleted file mode 100644 index 693905084151..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-3bb8fbd9e83703887e0a3c196031b931c0d8dbc6835dfac20107ea7412ce9fbb.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n proof_compression_jobs_fri (\n l1_batch_number,\n status,\n created_at,\n updated_at\n )\n VALUES\n ($1, $2, NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET status = $2\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Text" - ] - }, - "nullable": [] - }, - "hash": "3bb8fbd9e83703887e0a3c196031b931c0d8dbc6835dfac20107ea7412ce9fbb" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-434f7cb51a7d22948cd26e962679a67936d572f8046d3a1c7a4f100ff209d81d.json b/prover/crates/lib/prover_dal/.sqlx/query-434f7cb51a7d22948cd26e962679a67936d572f8046d3a1c7a4f100ff209d81d.json deleted file mode 100644 index 7615523f92f1..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-434f7cb51a7d22948cd26e962679a67936d572f8046d3a1c7a4f100ff209d81d.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE prover_jobs_fri SET status = $1\n WHERE l1_batch_number = $2\n AND sequence_number = $3\n AND aggregation_round = $4\n AND circuit_id = $5", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int8", - "Int4", - "Int2", - "Int2" - ] - }, - "nullable": [] - }, - "hash": "434f7cb51a7d22948cd26e962679a67936d572f8046d3a1c7a4f100ff209d81d" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-73266a8526c6adc315900e2e95441976a264759c4060c1a38e466ee2052fc17d.json b/prover/crates/lib/prover_dal/.sqlx/query-73266a8526c6adc315900e2e95441976a264759c4060c1a38e466ee2052fc17d.json new file mode 100644 index 000000000000..f8b141a8dac9 --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-73266a8526c6adc315900e2e95441976a264759c4060c1a38e466ee2052fc17d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n recursion_tip_witness_jobs_fri (l1_batch_number, status, number_of_final_node_jobs, created_at, updated_at)\n VALUES\n ($1, 'waiting_for_proofs', 1, NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET\n status = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "73266a8526c6adc315900e2e95441976a264759c4060c1a38e466ee2052fc17d" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-9730c8225ff2cf3111185e81f602a4a98ec63eb942c73ce4448d0957346047cd.json b/prover/crates/lib/prover_dal/.sqlx/query-9730c8225ff2cf3111185e81f602a4a98ec63eb942c73ce4448d0957346047cd.json new file mode 100644 index 000000000000..d23ed8d9fc8a --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-9730c8225ff2cf3111185e81f602a4a98ec63eb942c73ce4448d0957346047cd.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE leaf_aggregation_witness_jobs_fri\n SET\n status = $1,\n attempts = $2\n WHERE\n l1_batch_number = $3\n AND circuit_id = $4\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Int2", + "Int8", + "Int2" + ] + }, + "nullable": [] + }, + "hash": "9730c8225ff2cf3111185e81f602a4a98ec63eb942c73ce4448d0957346047cd" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-a817f0fec85388b3e2510ce259208a01b63ae4aa03c983c3a52c802d585e5a80.json b/prover/crates/lib/prover_dal/.sqlx/query-a817f0fec85388b3e2510ce259208a01b63ae4aa03c983c3a52c802d585e5a80.json new file mode 100644 index 000000000000..93532150f7f8 --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-a817f0fec85388b3e2510ce259208a01b63ae4aa03c983c3a52c802d585e5a80.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at)\n VALUES\n ($1, $2, NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET\n status = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Text" + ] + }, + "nullable": [] + }, + "hash": "a817f0fec85388b3e2510ce259208a01b63ae4aa03c983c3a52c802d585e5a80" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-aabcfa9005b8e1d84cfa083a47a700302981be0afef31a8864613484f8521f9e.json b/prover/crates/lib/prover_dal/.sqlx/query-aabcfa9005b8e1d84cfa083a47a700302981be0afef31a8864613484f8521f9e.json deleted file mode 100644 index 3d60050c92ed..000000000000 --- a/prover/crates/lib/prover_dal/.sqlx/query-aabcfa9005b8e1d84cfa083a47a700302981be0afef31a8864613484f8521f9e.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE prover_jobs_fri \n SET status = $1, attempts = $2\n WHERE l1_batch_number = $3\n AND sequence_number =$4\n AND aggregation_round = $5\n AND circuit_id = $6", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Int2", - "Int8", - "Int4", - "Int2", - "Int2" - ] - }, - "nullable": [] - }, - "hash": "aabcfa9005b8e1d84cfa083a47a700302981be0afef31a8864613484f8521f9e" -} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-c8daa62b3835c15fafb3f83deabb5a4672ad50a9de92c84d939ac4c69842e355.json b/prover/crates/lib/prover_dal/.sqlx/query-c8daa62b3835c15fafb3f83deabb5a4672ad50a9de92c84d939ac4c69842e355.json new file mode 100644 index 000000000000..cadc931fa1ca --- /dev/null +++ b/prover/crates/lib/prover_dal/.sqlx/query-c8daa62b3835c15fafb3f83deabb5a4672ad50a9de92c84d939ac4c69842e355.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO\n node_aggregation_witness_jobs_fri (l1_batch_number, circuit_id, status, created_at, updated_at)\n VALUES\n ($1, $2, 'waiting_for_proofs', NOW(), NOW())\n ON CONFLICT (l1_batch_number, circuit_id, depth) DO\n UPDATE\n SET\n status = $3\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int2", + "Text" + ] + }, + "nullable": [] + }, + "hash": "c8daa62b3835c15fafb3f83deabb5a4672ad50a9de92c84d939ac4c69842e355" +} diff --git a/prover/crates/lib/prover_dal/.sqlx/query-63cf7038e6c48af8ed9afc7d6ea07edd87cb16a79c13e7d4291d99736e51d3b9.json b/prover/crates/lib/prover_dal/.sqlx/query-e875dcbbdaed6998dbea45d4eab5d005d8760c4809b7aef902155196873da66e.json similarity index 82% rename from prover/crates/lib/prover_dal/.sqlx/query-63cf7038e6c48af8ed9afc7d6ea07edd87cb16a79c13e7d4291d99736e51d3b9.json rename to prover/crates/lib/prover_dal/.sqlx/query-e875dcbbdaed6998dbea45d4eab5d005d8760c4809b7aef902155196873da66e.json index 208b23d939f8..4ee9278fe42a 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-63cf7038e6c48af8ed9afc7d6ea07edd87cb16a79c13e7d4291d99736e51d3b9.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-e875dcbbdaed6998dbea45d4eab5d005d8760c4809b7aef902155196873da66e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n scheduler_witness_jobs_fri (\n l1_batch_number,\n scheduler_partial_input_blob_url,\n status,\n created_at,\n updated_at\n )\n VALUES\n ($1, '', 'waiting_for_proofs', NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET status = $2\n ", + "query": "\n INSERT INTO\n scheduler_witness_jobs_fri (\n l1_batch_number,\n scheduler_partial_input_blob_url,\n status,\n created_at,\n updated_at\n )\n VALUES\n ($1, '', 'waiting_for_proofs', NOW(), NOW())\n ON CONFLICT (l1_batch_number) DO\n UPDATE\n SET\n status = $2\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "63cf7038e6c48af8ed9afc7d6ea07edd87cb16a79c13e7d4291d99736e51d3b9" + "hash": "e875dcbbdaed6998dbea45d4eab5d005d8760c4809b7aef902155196873da66e" } diff --git a/prover/crates/lib/prover_dal/.sqlx/query-0eac6f7b2d799059328584029b437891598dc79b5ed11258b2c90c3f282929ad.json b/prover/crates/lib/prover_dal/.sqlx/query-eec29cbff034818f4fb5ec1e6ad38e1010d7389457b3c97e9b238a3a0291a54e.json similarity index 78% rename from prover/crates/lib/prover_dal/.sqlx/query-0eac6f7b2d799059328584029b437891598dc79b5ed11258b2c90c3f282929ad.json rename to prover/crates/lib/prover_dal/.sqlx/query-eec29cbff034818f4fb5ec1e6ad38e1010d7389457b3c97e9b238a3a0291a54e.json index 61518273b4d3..f8e92b1ad666 100644 --- a/prover/crates/lib/prover_dal/.sqlx/query-0eac6f7b2d799059328584029b437891598dc79b5ed11258b2c90c3f282929ad.json +++ b/prover/crates/lib/prover_dal/.sqlx/query-eec29cbff034818f4fb5ec1e6ad38e1010d7389457b3c97e9b238a3a0291a54e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n leaf_aggregation_witness_jobs_fri (\n l1_batch_number,\n circuit_id,\n status,\n number_of_basic_circuits,\n created_at,\n updated_at\n )\n VALUES\n ($1, $2, 'waiting_for_proofs', 2, NOW(), NOW())\n ON CONFLICT (l1_batch_number, circuit_id) DO\n UPDATE\n SET status = $3\n ", + "query": "\n INSERT INTO\n leaf_aggregation_witness_jobs_fri (\n l1_batch_number,\n circuit_id,\n status,\n number_of_basic_circuits,\n created_at,\n updated_at\n )\n VALUES\n ($1, $2, 'waiting_for_proofs', 2, NOW(), NOW())\n ON CONFLICT (l1_batch_number, circuit_id) DO\n UPDATE\n SET\n status = $3\n ", "describe": { "columns": [], "parameters": { @@ -12,5 +12,5 @@ }, "nullable": [] }, - "hash": "0eac6f7b2d799059328584029b437891598dc79b5ed11258b2c90c3f282929ad" + "hash": "eec29cbff034818f4fb5ec1e6ad38e1010d7389457b3c97e9b238a3a0291a54e" } diff --git a/prover/crates/lib/prover_dal/src/cli_test_dal.rs b/prover/crates/lib/prover_dal/src/cli_test_dal.rs index 19fe0e4f57b0..d08418203378 100644 --- a/prover/crates/lib/prover_dal/src/cli_test_dal.rs +++ b/prover/crates/lib/prover_dal/src/cli_test_dal.rs @@ -21,11 +21,16 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - "UPDATE prover_jobs_fri SET status = $1 - WHERE l1_batch_number = $2 + r#" + UPDATE prover_jobs_fri + SET + status = $1 + WHERE + l1_batch_number = $2 AND sequence_number = $3 AND aggregation_round = $4 - AND circuit_id = $5", + AND circuit_id = $5 + "#, status.to_string(), batch_number.0 as i64, sequence_number as i64, @@ -44,7 +49,7 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - " + r#" INSERT INTO leaf_aggregation_witness_jobs_fri ( l1_batch_number, @@ -58,8 +63,9 @@ impl CliTestDal<'_, '_> { ($1, $2, 'waiting_for_proofs', 2, NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id) DO UPDATE - SET status = $3 - ", + SET + status = $3 + "#, batch_number.0 as i64, circuit_id as i16, status.to_string() @@ -76,21 +82,16 @@ impl CliTestDal<'_, '_> { circuit_id: u8, ) { sqlx::query!( - " + r#" INSERT INTO - node_aggregation_witness_jobs_fri ( - l1_batch_number, - circuit_id, - status, - created_at, - updated_at - ) + node_aggregation_witness_jobs_fri (l1_batch_number, circuit_id, status, created_at, updated_at) VALUES ($1, $2, 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number, circuit_id, depth) DO UPDATE - SET status = $3 - ", + SET + status = $3 + "#, batch_number.0 as i64, circuit_id as i16, status.to_string(), @@ -102,21 +103,16 @@ impl CliTestDal<'_, '_> { pub async fn insert_rt_job(&mut self, status: WitnessJobStatus, batch_number: L1BatchNumber) { sqlx::query!( - " + r#" INSERT INTO - recursion_tip_witness_jobs_fri ( - l1_batch_number, - status, - number_of_final_node_jobs, - created_at, - updated_at - ) + recursion_tip_witness_jobs_fri (l1_batch_number, status, number_of_final_node_jobs, created_at, updated_at) VALUES - ($1, 'waiting_for_proofs',1, NOW(), NOW()) + ($1, 'waiting_for_proofs', 1, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -131,7 +127,7 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - " + r#" INSERT INTO scheduler_witness_jobs_fri ( l1_batch_number, @@ -144,8 +140,9 @@ impl CliTestDal<'_, '_> { ($1, '', 'waiting_for_proofs', NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -160,20 +157,16 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - " + r#" INSERT INTO - proof_compression_jobs_fri ( - l1_batch_number, - status, - created_at, - updated_at - ) + proof_compression_jobs_fri (l1_batch_number, status, created_at, updated_at) VALUES ($1, $2, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO UPDATE - SET status = $2 - ", + SET + status = $2 + "#, batch_number.0 as i64, status.to_string(), ) @@ -192,12 +185,17 @@ impl CliTestDal<'_, '_> { sequence_number: usize, ) { sqlx::query!( - "UPDATE prover_jobs_fri - SET status = $1, attempts = $2 - WHERE l1_batch_number = $3 - AND sequence_number =$4 + r#" + UPDATE prover_jobs_fri + SET + status = $1, + attempts = $2 + WHERE + l1_batch_number = $3 + AND sequence_number = $4 AND aggregation_round = $5 - AND circuit_id = $6", + AND circuit_id = $6 + "#, status.to_string(), attempts as i64, batch_number.0 as i64, @@ -218,10 +216,15 @@ impl CliTestDal<'_, '_> { batch_number: L1BatchNumber, ) { sqlx::query!( - "UPDATE leaf_aggregation_witness_jobs_fri - SET status = $1, attempts = $2 - WHERE l1_batch_number = $3 - AND circuit_id = $4", + r#" + UPDATE leaf_aggregation_witness_jobs_fri + SET + status = $1, + attempts = $2 + WHERE + l1_batch_number = $3 + AND circuit_id = $4 + "#, status.to_string(), attempts as i64, batch_number.0 as i64, diff --git a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs index 1a3b8de0ce4b..71d0c11728b1 100644 --- a/prover/crates/lib/prover_dal/src/fri_prover_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_prover_dal.rs @@ -528,8 +528,14 @@ impl FriProverDal<'_, '_> { SELECT protocol_version AS "protocol_version!", protocol_version_patch AS "protocol_version_patch!", - COUNT(*) FILTER (WHERE status = 'queued') as queued, - COUNT(*) FILTER (WHERE status = 'in_progress') as in_progress + COUNT(*) FILTER ( + WHERE + status = 'queued' + ) AS queued, + COUNT(*) FILTER ( + WHERE + status = 'in_progress' + ) AS in_progress FROM prover_jobs_fri WHERE diff --git a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs index 66e34f7f8e75..c7ba0f60ef3f 100644 --- a/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/crates/lib/prover_dal/src/fri_witness_generator_dal.rs @@ -1719,7 +1719,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, @@ -1786,7 +1789,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, @@ -1827,7 +1833,10 @@ impl FriWitnessGeneratorDal<'_, '_> { WHERE l1_batch_number = $1 AND attempts >= $2 - AND (status = 'in_progress' OR status = 'failed') + AND ( + status = 'in_progress' + OR status = 'failed' + ) RETURNING l1_batch_number, status, From 790e1a5683f8810efc7f2538f886cc00380e3b57 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 24 Sep 2024 12:46:14 +0200 Subject: [PATCH 41/51] Fix Signed-off-by: Danil --- .../src/jest-setup/global-setup.ts | 6 +- core/tests/ts-integration/src/utils.ts | 70 ++++++++++--------- core/tests/ts-integration/tests/fees.test.ts | 54 +++++++------- etc/utils/src/kill.ts | 8 ++- 4 files changed, 71 insertions(+), 67 deletions(-) diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index 37e6b69ec836..a7bb0edcef14 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -3,7 +3,7 @@ import { bigIntReplacer } from '../helpers'; import { TestContextOwner, loadTestEnvironment } from '../index'; import path from 'path'; import { runServerInBackground } from '../utils'; -import utils from 'utils'; +import { exec } from 'utils'; declare global { var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; @@ -30,9 +30,9 @@ async function performSetup(_globalConfig: any, _projectConfig: any) { if (autoKill) { try { - await utils.exec(`killall -KILL zksync_server`); + await exec(`killall -KILL zksync_server`); } catch (err) { - console.log(`ignored error: ${err}`); + console.log(`ignored error: ${ err }`); } } diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index 627867c214c0..134a67ae44c7 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -17,28 +17,28 @@ import { // executes a command in background and returns a child process handle // by default pipes data to parent's stdio but this can be overridden export function background({ - command, - stdio = 'inherit', - cwd, - env -}: { + command, + stdio = 'inherit', + cwd, + env + }: { command: string; stdio: any; cwd?: ProcessEnvOptions['cwd']; env?: ProcessEnvOptions['env']; }): ChildProcessWithoutNullStreams { command = command.replace(/\n/g, ' '); - console.log(`Run command ${command}`); - return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env }); + console.log(`Run command ${ command }`); + return _spawn(command, {stdio: stdio, shell: true, detached: true, cwd, env}); } export function runInBackground({ - command, - components, - stdio, - cwd, - env -}: { + command, + components, + stdio, + cwd, + env + }: { command: string; components?: string[]; stdio: any; @@ -46,19 +46,19 @@ export function runInBackground({ env?: Parameters[0]['env']; }): ChildProcessWithoutNullStreams { if (components && components.length > 0) { - command += ` --components=${components.join(',')}`; + command += ` --components=${ components.join(',') }`; } - return background({ command, stdio, cwd, env }); + return background({command, stdio, cwd, env}); } export function runServerInBackground({ - components, - stdio, - cwd, - env, - useZkInception, - chain -}: { + components, + stdio, + cwd, + env, + useZkInception, + chain + }: { components?: string[]; stdio: any; cwd?: Parameters[0]['cwd']; @@ -72,12 +72,12 @@ export function runServerInBackground({ if (useZkInception) { command = 'zk_inception server'; if (chain) { - command += ` --chain ${chain}`; + command += ` --chain ${ chain }`; } } else { command = 'zk server'; } - return runInBackground({ command, components, stdio, cwd, env }); + return runInBackground({command, components, stdio, cwd, env}); } export interface MainNodeSpawnOptions { @@ -97,13 +97,14 @@ export class Node { public readonly tester: Tester, public proc: ChildProcessWithoutNullStreams, private readonly type: TYPE - ) {} + ) { + } public async terminate() { try { await killPidWithAllChilds(this.proc.pid!, 9); } catch (err) { - console.log(`ignored error: ${err}`); + console.log(`ignored error: ${ err }`); } } @@ -114,9 +115,9 @@ export class Node { */ public static async killAll(type: NodeType) { try { - await utils.exec(`killall -KILL ${type}`); + await utils.exec(`killall -KILL ${ type }`); } catch (err) { - console.log(`ignored error: ${err}`); + console.log(`ignored error: ${ err }`); } } @@ -143,7 +144,7 @@ export class Node { } } // It's going to panic anyway, since the server is a singleton entity, so better to exit early. - throw new Error(`${this.type} didn't stop after a kill request`); + throw new Error(`${ this.type } didn't stop after a kill request`); } } @@ -154,11 +155,12 @@ export class NodeSpawner { private readonly fileConfig: FileConfig, private readonly options: MainNodeSpawnOptions, private readonly env?: ProcessEnvOptions['env'] - ) {} + ) { + } public async spawnMainNode(newL1GasPrice?: string, newPubdataPrice?: string): Promise> { const env = this.env ?? process.env; - const { fileConfig, pathToHome, options, logs } = this; + const {fileConfig, pathToHome, options, logs} = this; const testMode = newPubdataPrice || newL1GasPrice; @@ -228,13 +230,13 @@ async function waitForNodeToStart(tester: Tester, proc: ChildProcessWithoutNullS while (true) { try { const blockNumber = await tester.syncWallet.provider.getBlockNumber(); - console.log(`Initialized node API on ${l2Url}; latest block: ${blockNumber}`); + console.log(`Initialized node API on ${ l2Url }; latest block: ${ blockNumber }`); break; } catch (err) { if (proc.exitCode != null) { - assert.fail(`server failed to start, exitCode = ${proc.exitCode}`); + assert.fail(`server failed to start, exitCode = ${ proc.exitCode }`); } - console.log(`Node waiting for API on ${l2Url}`); + console.log(`Node waiting for API on ${ l2Url }`); await utils.sleep(1); } } diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 4a8d778d4429..8ca8399ddf35 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; @@ -79,7 +79,7 @@ testFees('Test fees', function () { try { await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); } catch (err) { - console.log(`ignored error: ${err}`); + console.log(`ignored error: ${ err }`); } // TODO: imporove the waiting logic @@ -114,7 +114,7 @@ testFees('Test fees', function () { const pathToMainLogs = await logsPath('server.log'); mainLogs = await fs.open(pathToMainLogs, 'a'); - console.log(`Writing server logs to ${pathToMainLogs}`); + console.log(`Writing server logs to ${ pathToMainLogs }`); mainNodeSpawner = new NodeSpawner(pathToHome, mainLogs, fileConfig, { enableConsensus, @@ -212,7 +212,7 @@ testFees('Test fees', function () { ); } - console.log(`Full report: \n\n${reports.join('\n\n')}`); + console.log(`Full report: \n\n${ reports.join('\n\n') }`); }); test('Test gas consumption under large L1 gas price', async () => { @@ -245,7 +245,7 @@ testFees('Test fees', function () { // Firstly, let's test a successful transaction. const largeData = ethers.randomBytes(90_000); - const tx = await l1Messenger.sendToL1(largeData, { type: 0 }); + const tx = await l1Messenger.sendToL1(largeData, {type: 0}); expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); const receipt = await tx.wait(); expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); @@ -256,7 +256,7 @@ testFees('Test fees', function () { const systemContextGasPerPubdataByte = await systemContext.gasPerPubdataByte(); expect(systemContextGasPerPubdataByte).toEqual(MAX_GAS_PER_PUBDATA); - const dataHash = await l1Messenger.sendToL1.staticCall(largeData, { type: 0 }); + const dataHash = await l1Messenger.sendToL1.staticCall(largeData, {type: 0}); expect(dataHash).toEqual(ethers.keccak256(largeData)); // Secondly, let's test an unsuccessful transaction with large refund. @@ -332,7 +332,7 @@ async function updateReport( const balanceBefore = await sender.getBalance(); const transaction = await sender.sendTransaction(transactionRequest); - console.log(`Sending transaction: ${transaction.hash}`); + console.log(`Sending transaction: ${ transaction.hash }`); await transaction.wait(); const balanceAfter = await sender.getBalance(); const balanceDiff = balanceBefore - balanceAfter; @@ -340,12 +340,12 @@ async function updateReport( const l2PriceAsNumber = +ethers.formatEther(balanceDiff); const l2EstimatedPriceAsNumber = +ethers.formatEther(estimatedPrice); - const gasReport = `Gas price ${newL1GasPrice / 1000000000n} gwei: - L1 cost ${expectedL1Price}, - L2 estimated cost: ${l2EstimatedPriceAsNumber} - Estimated Gain: ${expectedL1Price / l2EstimatedPriceAsNumber} - L2 cost: ${l2PriceAsNumber}, - Gain: ${expectedL1Price / l2PriceAsNumber}\n`; + const gasReport = `Gas price ${ newL1GasPrice / 1000000000n } gwei: + L1 cost ${ expectedL1Price }, + L2 estimated cost: ${ l2EstimatedPriceAsNumber } + Estimated Gain: ${ expectedL1Price / l2EstimatedPriceAsNumber } + L2 cost: ${ l2PriceAsNumber }, + Gain: ${ expectedL1Price / l2PriceAsNumber }\n`; console.log(gasReport); return oldReport + gasReport; diff --git a/etc/utils/src/kill.ts b/etc/utils/src/kill.ts index 7fdab85afadd..d6da3cc2aed8 100644 --- a/etc/utils/src/kill.ts +++ b/etc/utils/src/kill.ts @@ -3,17 +3,19 @@ import { exec } from 'node:child_process'; export async function killPidWithAllChilds(pid: number, signalNumber: number) { let childs = [pid]; + console.log(pid, signalNumber); while (true) { try { let child = childs.at(-1); - childs.push(+(await promisify(exec)(`pgrep -P ${child}`)).stdout); + console.log(child); + childs.push(+(await promisify(exec)(`pgrep -P ${ child }`)).stdout); } catch (e) { break; } } // We always run the test using additional tools, that means we have to kill not the main process, but the child process for (let i = childs.length - 1; i >= 0; i--) { - console.log(`kill ${childs[i]}`); - await promisify(exec)(`kill -${signalNumber} ${childs[i]}`); + console.log(`kill ${ childs[i] }`); + await promisify(exec)(`kill -${ signalNumber } ${ childs[i] }`); } } From f5cfc0d53ac04b0b07fded9fcde0439e8dfd391c Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 24 Sep 2024 15:58:40 +0200 Subject: [PATCH 42/51] fix Signed-off-by: Danil --- core/tests/ts-integration/tests/fees.test.ts | 298 +++++++++---------- core/tests/ts-integration/tests/utils.ts | 12 +- etc/utils/src/kill.ts | 8 +- 3 files changed, 161 insertions(+), 157 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 8ca8399ddf35..dad3c23558b8 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -76,15 +76,15 @@ testFees('Test fees', function () { beforeAll(async () => { testMaster = TestMaster.getInstance(__filename); - try { - await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); - } catch (err) { - console.log(`ignored error: ${ err }`); - } + // try { + // await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); + // } catch (err) { + // console.log(`ignored error: ${ err }`); + // } // TODO: imporove the waiting logic - console.log('Waiting for the server to stop...'); - await utils.sleep(10); + // console.log('Waiting for the server to stop...'); + // await utils.sleep(10); if (!fileConfig.loadFromFile) { ethClientWeb3Url = process.env.ETH_CLIENT_WEB3_URL!; @@ -130,152 +130,152 @@ testFees('Test fees', function () { aliceErc20 = new ethers.Contract(tokenDetails.l1Address, zksync.utils.IERC20, alice.ethWallet()); }); - test('Test fees', async () => { - const receiver = ethers.Wallet.createRandom().address; - - // Getting ETH price in gas. - const feeTestL1Receipt = await ( - await alice.ethWallet().sendTransaction({ - to: receiver, - value: 1n - }) - ).wait(); - - if (feeTestL1Receipt === null) { - throw new Error('Failed to send ETH transaction'); - } - - const feeTestL1ReceiptERC20 = await ( - await alice.ethWallet().sendTransaction({ - to: aliceErc20.getAddress(), - data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]) - }) - ).wait(); - - if (feeTestL1ReceiptERC20 === null) { - throw new Error('Failed to send ERC20 transaction'); - } - - // Warming up slots for the receiver - await ( - await alice.sendTransaction({ - to: receiver, - value: BigInt(1) - }) - ).wait(); - - await ( - await alice.sendTransaction({ - data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), - to: tokenDetails.l2Address - }) - ).wait(); - - let reports = [ - 'ETH transfer (to new):\n\n', - 'ETH transfer (to old):\n\n', - 'ERC20 transfer (to new):\n\n', - 'ERC20 transfer (to old):\n\n' - ]; - for (const gasPrice of L1_GAS_PRICES_TO_TEST) { - // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. - await mainNode.killAndWaitForShutdown(); - mainNode = await mainNodeSpawner.spawnMainNode(gasPrice.toString(), gasPrice.toString()); - - reports = await appendResults( - alice, - [feeTestL1Receipt, feeTestL1Receipt, feeTestL1ReceiptERC20, feeTestL1ReceiptERC20], - // We always regenerate new addresses for transaction requests in order to estimate the cost for a new account - [ - { - to: ethers.Wallet.createRandom().address, - value: 1n - }, - { - to: receiver, - value: 1n - }, - { - data: aliceErc20.interface.encodeFunctionData('transfer', [ - ethers.Wallet.createRandom().address, - 1n - ]), - to: tokenDetails.l2Address - }, - { - data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), - to: tokenDetails.l2Address - } - ], - gasPrice, - reports - ); - } - - console.log(`Full report: \n\n${ reports.join('\n\n') }`); + test('Test all fees', async () => { + // const receiver = ethers.Wallet.createRandom().address; + // + // // Getting ETH price in gas. + // const feeTestL1Receipt = await ( + // await alice.ethWallet().sendTransaction({ + // to: receiver, + // value: 1n + // }) + // ).wait(); + // + // if (feeTestL1Receipt === null) { + // throw new Error('Failed to send ETH transaction'); + // } + // + // const feeTestL1ReceiptERC20 = await ( + // await alice.ethWallet().sendTransaction({ + // to: aliceErc20.getAddress(), + // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]) + // }) + // ).wait(); + // + // if (feeTestL1ReceiptERC20 === null) { + // throw new Error('Failed to send ERC20 transaction'); + // } + // + // // Warming up slots for the receiver + // await ( + // await alice.sendTransaction({ + // to: receiver, + // value: BigInt(1) + // }) + // ).wait(); + // + // await ( + // await alice.sendTransaction({ + // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), + // to: tokenDetails.l2Address + // }) + // ).wait(); + // + // let reports = [ + // 'ETH transfer (to new):\n\n', + // 'ETH transfer (to old):\n\n', + // 'ERC20 transfer (to new):\n\n', + // 'ERC20 transfer (to old):\n\n' + // ]; + // for (const gasPrice of L1_GAS_PRICES_TO_TEST) { + // // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. + // await mainNode.killAndWaitForShutdown(); + // mainNode = await mainNodeSpawner.spawnMainNode(gasPrice.toString(), gasPrice.toString()); + // + // reports = await appendResults( + // alice, + // [feeTestL1Receipt, feeTestL1Receipt, feeTestL1ReceiptERC20, feeTestL1ReceiptERC20], + // // We always regenerate new addresses for transaction requests in order to estimate the cost for a new account + // [ + // { + // to: ethers.Wallet.createRandom().address, + // value: 1n + // }, + // { + // to: receiver, + // value: 1n + // }, + // { + // data: aliceErc20.interface.encodeFunctionData('transfer', [ + // ethers.Wallet.createRandom().address, + // 1n + // ]), + // to: tokenDetails.l2Address + // }, + // { + // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), + // to: tokenDetails.l2Address + // } + // ], + // gasPrice, + // reports + // ); + // } + // + // console.log(`Full report: \n\n${ reports.join('\n\n') }`); }); test('Test gas consumption under large L1 gas price', async () => { - if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { - // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. - return; - } - - // In this test we check that the server works fine when the required gasLimit is over u32::MAX. - // Under normal server behavior, the maximal gas spent on pubdata is around 120kb * 2^20 gas/byte = ~120 * 10^9 gas. - - // In this test we will set gas per pubdata byte to its maximum value, while publishing a large L1->L2 message. - - const minimalL2GasPrice = testMaster.environment().minimalL2GasPrice; - - // We want the total gas limit to be over u32::MAX, so we need the gas per pubdata to be 50k. + // if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { + // // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. + // return; + // } // - // Note, that in case, any sort of overhead is present in the l2 fair gas price calculation, the final - // gas per pubdata may be lower than 50_000. Here we assume that it is not the case, but we'll double check - // that the gasLimit is indeed over u32::MAX, which is the most important tested property. - const requiredPubdataPrice = minimalL2GasPrice * 100_000n; - - await mainNode.killAndWaitForShutdown(); - mainNode = await mainNodeSpawner.spawnMainNode( - requiredPubdataPrice.toString(), - requiredPubdataPrice.toString() - ); - - const l1Messenger = new ethers.Contract(zksync.utils.L1_MESSENGER_ADDRESS, zksync.utils.L1_MESSENGER, alice); - - // Firstly, let's test a successful transaction. - const largeData = ethers.randomBytes(90_000); - const tx = await l1Messenger.sendToL1(largeData, {type: 0}); - expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); - const receipt = await tx.wait(); - expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); - - // Let's also check that the same transaction would work as eth_call - const systemContextArtifact = getTestContract('ISystemContext'); - const systemContext = new ethers.Contract(SYSTEM_CONTEXT_ADDRESS, systemContextArtifact.abi, alice.provider); - const systemContextGasPerPubdataByte = await systemContext.gasPerPubdataByte(); - expect(systemContextGasPerPubdataByte).toEqual(MAX_GAS_PER_PUBDATA); - - const dataHash = await l1Messenger.sendToL1.staticCall(largeData, {type: 0}); - expect(dataHash).toEqual(ethers.keccak256(largeData)); - - // Secondly, let's test an unsuccessful transaction with large refund. - - // The size of the data has increased, so the previous gas limit is not enough. - const largerData = ethers.randomBytes(91_000); - const gasToPass = receipt.gasUsed; - const unsuccessfulTx = await l1Messenger.sendToL1(largerData, { - gasLimit: gasToPass, - type: 0 - }); - - try { - await unsuccessfulTx.wait(); - throw new Error('The transaction should have reverted'); - } catch { - const receipt = await alice.provider.getTransactionReceipt(unsuccessfulTx.hash); - expect(gasToPass - receipt!.gasUsed > UINT32_MAX).toBeTruthy(); - } + // // In this test we check that the server works fine when the required gasLimit is over u32::MAX. + // // Under normal server behavior, the maximal gas spent on pubdata is around 120kb * 2^20 gas/byte = ~120 * 10^9 gas. + // + // // In this test we will set gas per pubdata byte to its maximum value, while publishing a large L1->L2 message. + // + // const minimalL2GasPrice = testMaster.environment().minimalL2GasPrice; + // + // // We want the total gas limit to be over u32::MAX, so we need the gas per pubdata to be 50k. + // // + // // Note, that in case, any sort of overhead is present in the l2 fair gas price calculation, the final + // // gas per pubdata may be lower than 50_000. Here we assume that it is not the case, but we'll double check + // // that the gasLimit is indeed over u32::MAX, which is the most important tested property. + // const requiredPubdataPrice = minimalL2GasPrice * 100_000n; + // + // await mainNode.killAndWaitForShutdown(); + // mainNode = await mainNodeSpawner.spawnMainNode( + // requiredPubdataPrice.toString(), + // requiredPubdataPrice.toString() + // ); + // + // const l1Messenger = new ethers.Contract(zksync.utils.L1_MESSENGER_ADDRESS, zksync.utils.L1_MESSENGER, alice); + // + // // Firstly, let's test a successful transaction. + // const largeData = ethers.randomBytes(90_000); + // const tx = await l1Messenger.sendToL1(largeData, {type: 0}); + // expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); + // const receipt = await tx.wait(); + // expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); + // + // // Let's also check that the same transaction would work as eth_call + // const systemContextArtifact = getTestContract('ISystemContext'); + // const systemContext = new ethers.Contract(SYSTEM_CONTEXT_ADDRESS, systemContextArtifact.abi, alice.provider); + // const systemContextGasPerPubdataByte = await systemContext.gasPerPubdataByte(); + // expect(systemContextGasPerPubdataByte).toEqual(MAX_GAS_PER_PUBDATA); + // + // const dataHash = await l1Messenger.sendToL1.staticCall(largeData, {type: 0}); + // expect(dataHash).toEqual(ethers.keccak256(largeData)); + // + // // Secondly, let's test an unsuccessful transaction with large refund. + // + // // The size of the data has increased, so the previous gas limit is not enough. + // const largerData = ethers.randomBytes(91_000); + // const gasToPass = receipt.gasUsed; + // const unsuccessfulTx = await l1Messenger.sendToL1(largerData, { + // gasLimit: gasToPass, + // type: 0 + // }); + // + // try { + // await unsuccessfulTx.wait(); + // throw new Error('The transaction should have reverted'); + // } catch { + // const receipt = await alice.provider.getTransactionReceipt(unsuccessfulTx.hash); + // expect(gasToPass - receipt!.gasUsed > UINT32_MAX).toBeTruthy(); + // } }); afterAll(async () => { diff --git a/core/tests/ts-integration/tests/utils.ts b/core/tests/ts-integration/tests/utils.ts index 72669cdf4555..ead050437c61 100644 --- a/core/tests/ts-integration/tests/utils.ts +++ b/core/tests/ts-integration/tests/utils.ts @@ -30,8 +30,8 @@ function setPropertyInGeneralConfig(pathToHome: string, fileConfig: any, propert }); const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); - const regex = new RegExp(`${property}:\\s*\\d+(\\.\\d+)?`, 'g'); - const newGeneralConfig = generalConfig.replace(regex, `${property}: ${value}`); + const regex = new RegExp(`${ property }:\\s*\\d+(\\.\\d+)?`, 'g'); + const newGeneralConfig = generalConfig.replace(regex, `${ property }: ${ value }`); fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); } @@ -46,17 +46,17 @@ function setGasAdjusterProperty(pathToHome: string, fileConfig: any, property: s const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); // Define the regex pattern to check if the property already exists - const propertyRegex = new RegExp(`(^\\s*${property}:\\s*\\d+(\\.\\d+)?$)`, 'm'); + const propertyRegex = new RegExp(`(^\\s*${ property }:\\s*\\d+(\\.\\d+)?$)`, 'm'); const gasAdjusterRegex = new RegExp('(^\\s*gas_adjuster:.*$)', 'gm'); let newGeneralConfig; if (propertyRegex.test(generalConfig)) { // If the property exists, modify its value - newGeneralConfig = generalConfig.replace(propertyRegex, `${property}: ${value}`); + newGeneralConfig = generalConfig.replace(propertyRegex, ` ${ property }: ${ value }`); } else { // If the property does not exist, add it under the gas_adjuster section - newGeneralConfig = generalConfig.replace(gasAdjusterRegex, `$1\n ${property}: ${value}`); + newGeneralConfig = generalConfig.replace(gasAdjusterRegex, `$1\n ${ property }: ${ value }`); } fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); @@ -72,7 +72,7 @@ function deleteProperty(pathToHome: string, fileConfig: any, property: string) { const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); // Define the regex pattern to find the property line and remove it completely - const propertyRegex = new RegExp(`^\\s*${property}:.*\\n?`, 'm'); + const propertyRegex = new RegExp(`^\\s*${ property }:.*\\n?`, 'm'); // Remove the line if the property exists const newGeneralConfig = generalConfig.replace(propertyRegex, ''); diff --git a/etc/utils/src/kill.ts b/etc/utils/src/kill.ts index d6da3cc2aed8..a0932c1acf62 100644 --- a/etc/utils/src/kill.ts +++ b/etc/utils/src/kill.ts @@ -7,15 +7,19 @@ export async function killPidWithAllChilds(pid: number, signalNumber: number) { while (true) { try { let child = childs.at(-1); - console.log(child); childs.push(+(await promisify(exec)(`pgrep -P ${ child }`)).stdout); } catch (e) { break; } } // We always run the test using additional tools, that means we have to kill not the main process, but the child process + console.log("childs", childs); for (let i = childs.length - 1; i >= 0; i--) { console.log(`kill ${ childs[i] }`); - await promisify(exec)(`kill -${ signalNumber } ${ childs[i] }`); + try { + await promisify(exec)(`kill -${ signalNumber } ${ childs[i] }`); + } catch (e) { + console.log(`Failed to kill ${ childs[i] } with ${ e }`,) + } } } From f70e7d8019ccbf855db562fb7ddebb9fcc44b8dd Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 25 Sep 2024 17:26:32 +0200 Subject: [PATCH 43/51] Correct kill the process Signed-off-by: Danil --- .github/workflows/ci-core-reusable.yml | 6 +- core/tests/ts-integration/src/env.ts | 55 ++- .../src/jest-setup/global-setup.ts | 33 +- core/tests/ts-integration/src/tester.ts | 75 ---- core/tests/ts-integration/src/types.ts | 4 +- core/tests/ts-integration/src/utils.ts | 96 +++-- core/tests/ts-integration/tests/fees.test.ts | 350 +++++++++--------- core/tests/ts-integration/tests/utils.ts | 12 +- etc/utils/src/kill.ts | 10 +- .../src/commands/test/args/fees.rs | 4 +- .../zk_supervisor/src/commands/test/fees.rs | 4 +- 11 files changed, 287 insertions(+), 362 deletions(-) delete mode 100644 core/tests/ts-integration/src/tester.ts diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index a816328eb56b..464eef067680 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -460,8 +460,10 @@ jobs: - name: Fee projection tests run: | ci_run killall -INT zksync_server || true - - ci_run zk_supervisor test fees --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log + ci_run zk_supervisor test fees --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log + ci_run zk_supervisor test fees --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log + ci_run zk_supervisor test fees --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log + ci_run zk_supervisor test fees --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log - name: Run revert tests run: | diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index 1a58d55d4775..18d9c084f78a 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -6,7 +6,17 @@ import { DataAvailabityMode, NodeMode, TestEnvironment } from './types'; import { Reporter } from './reporter'; import * as yaml from 'yaml'; import { L2_BASE_TOKEN_ADDRESS } from 'zksync-ethers/build/utils'; -import { loadConfig, loadEcosystem, shouldLoadConfigFromFile } from 'utils/build/file-configs'; +import { FileConfig, loadConfig, loadEcosystem, shouldLoadConfigFromFile } from 'utils/build/file-configs'; +import { NodeSpawner } from './utils'; +import { logsTestPath } from 'utils/build/logs'; +import * as nodefs from 'node:fs/promises'; +import { exec } from 'utils'; + +const enableConsensus = process.env.ENABLE_CONSENSUS === 'true'; + +async function logsPath(chain: string, name: string): Promise { + return await logsTestPath(chain, 'logs/server/', name); +} /** * Attempts to connect to server. @@ -60,8 +70,10 @@ function getMainWalletPk(pathToHome: string): string { /* Loads the environment for file based configs. */ -async function loadTestEnvironmentFromFile(chain: string): Promise { +async function loadTestEnvironmentFromFile(fileConfig: FileConfig): Promise { + let chain = fileConfig.chain!; const pathToHome = path.join(__dirname, '../../../..'); + let spawnNode = process.env.SPAWN_NODE; let nodeMode; if (process.env.EXTERNAL_NODE == 'true') { nodeMode = NodeMode.External; @@ -75,18 +87,42 @@ async function loadTestEnvironmentFromFile(chain: string): Promise { - const { loadFromFile, chain } = shouldLoadConfigFromFile(); + const fileConfig = shouldLoadConfigFromFile(); - if (loadFromFile) { - return await loadTestEnvironmentFromFile(chain); + if (fileConfig.loadFromFile) { + return await loadTestEnvironmentFromFile(fileConfig); } return await loadTestEnvironmentFromEnv(); } @@ -260,7 +295,7 @@ export async function loadTestEnvironmentFromEnv(): Promise { network, mainWalletPK, l2NodeUrl, - l2NodePid, + l2Node: undefined, l1NodeUrl, wsL2NodeUrl, healthcheckPort, diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index a7bb0edcef14..d050ae6705dd 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -2,8 +2,10 @@ import { shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { bigIntReplacer } from '../helpers'; import { TestContextOwner, loadTestEnvironment } from '../index'; import path from 'path'; -import { runServerInBackground } from '../utils'; +import { NodeSpawner, runServerInBackground } from '../utils'; import { exec } from 'utils'; +import fs from 'node:fs/promises'; +import { logsTestPath } from 'utils/build/logs'; declare global { var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; @@ -22,36 +24,7 @@ async function performSetup(_globalConfig: any, _projectConfig: any) { console.log(''); globalThis.rawWriteToConsole = console.log; - // Before starting any actual logic, we need to ensure that the server is running (it may not - // be the case, for example, right after deployment on stage). - const fileConfig = shouldLoadConfigFromFile(); - const pathToHome = path.join(__dirname, '../../../../..'); - const autoKill: boolean = !fileConfig.loadFromFile || !process.env.NO_KILL; - - if (autoKill) { - try { - await exec(`killall -KILL zksync_server`); - } catch (err) { - console.log(`ignored error: ${ err }`); - } - } - - let components = 'api,tree,eth,state_keeper,da_dispatcher,vm_runner_protective_reads'; - const env = process.env; - - let proc = runServerInBackground({ - components: [components], - stdio: ['ignore', 'ignore', 'ignore'], - cwd: pathToHome, - env: env, - useZkInception: fileConfig.loadFromFile, - chain: fileConfig.chain - }); - - proc.unref(); - let testEnvironment = await loadTestEnvironment(); - testEnvironment.l2NodePid = proc.pid!; const testContextOwner = new TestContextOwner(testEnvironment); const testContext = await testContextOwner.setupContext(); diff --git a/core/tests/ts-integration/src/tester.ts b/core/tests/ts-integration/src/tester.ts deleted file mode 100644 index cf2d84f95023..000000000000 --- a/core/tests/ts-integration/src/tester.ts +++ /dev/null @@ -1,75 +0,0 @@ -import * as ethers from 'ethers'; -import * as zksync from 'zksync-ethers'; -import * as fs from 'fs'; -import * as path from 'path'; - -export class Tester { - public runningFee: Map; - - constructor( - public ethProvider: ethers.Provider, - public ethWallet: ethers.Wallet, - public syncWallet: zksync.Wallet, - public web3Provider: zksync.Provider, - public isETHBasedChain: boolean, - public baseTokenAddress: string - ) { - this.runningFee = new Map(); - } - - // prettier-ignore - static async init(l1_rpc_addr: string, l2_rpc_addr: string, baseTokenAddress: string): Promise { - const ethProvider = new ethers.JsonRpcProvider(l1_rpc_addr); - ethProvider.pollingInterval = 100; - - const testConfigPath = path.join(process.env.ZKSYNC_HOME!, `etc/test_config/constant`); - const ethTestConfig = JSON.parse(fs.readFileSync(`${testConfigPath}/eth.json`, { encoding: 'utf-8' })); - - let ethWalletPK: string; - if (process.env.MASTER_WALLET_PK) { - ethWalletPK = process.env.MASTER_WALLET_PK; - } else { - const ethWalletHD = ethers.HDNodeWallet.fromMnemonic( - ethers.Mnemonic.fromPhrase(ethTestConfig.test_mnemonic), - "m/44'/60'/0'/0/0" - ); - - ethWalletPK = ethWalletHD.privateKey - } - - const ethWallet = new ethers.Wallet(ethWalletPK, ethProvider); - - const web3Provider = new zksync.Provider(l2_rpc_addr); - web3Provider.pollingInterval = 100; // It's OK to keep it low even on stage. - const syncWallet = new zksync.Wallet(ethWallet.privateKey, web3Provider, ethProvider); - - - // Since some tx may be pending on stage, we don't want to get stuck because of it. - // In order to not get stuck transactions, we manually cancel all the pending txs. - const latestNonce = await ethWallet.getNonce('latest'); - const pendingNonce = await ethWallet.getNonce('pending'); - const cancellationTxs = []; - for (let nonce = latestNonce; nonce != pendingNonce; nonce++) { - // For each transaction to override it, we need to provide greater fee. - // We would manually provide a value high enough (for a testnet) to be both valid - // and higher than the previous one. It's OK as we'll only be charged for the bass fee - // anyways. We will also set the miner's tip to 5 gwei, which is also much higher than the normal one. - const maxFeePerGas = ethers.parseEther("0.00000025"); // 250 gwei - const maxPriorityFeePerGas = ethers.parseEther("0.000000005"); // 5 gwei - cancellationTxs.push(ethWallet.sendTransaction({ - to: ethWallet.address, - nonce, - maxFeePerGas, - maxPriorityFeePerGas - }).then((tx) => tx.wait())); - } - if (cancellationTxs.length > 0) { - await Promise.all(cancellationTxs); - console.log(`Canceled ${cancellationTxs.length} pending transactions`); - } - - const isETHBasedChain = baseTokenAddress == zksync.utils.ETH_ADDRESS_IN_CONTRACTS; - - return new Tester(ethProvider, ethWallet, syncWallet, web3Provider, isETHBasedChain, baseTokenAddress); - } -} diff --git a/core/tests/ts-integration/src/types.ts b/core/tests/ts-integration/src/types.ts index 222f39d36bf9..ffb94ca32545 100644 --- a/core/tests/ts-integration/src/types.ts +++ b/core/tests/ts-integration/src/types.ts @@ -1,3 +1,5 @@ +import { Node, NodeType } from './utils'; + export enum NodeMode { Main, External @@ -58,7 +60,7 @@ export interface TestEnvironment { /* * L2 node PID */ - l2NodePid: number; + l2Node: Node | undefined; /** * Plaintext name of the L1 network name (i.e. `localhost` or `goerli`). */ diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index 134a67ae44c7..9d841792dbaf 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -1,7 +1,6 @@ import { spawn as _spawn, ChildProcessWithoutNullStreams, type ProcessEnvOptions } from 'child_process'; import { assert } from 'chai'; import { FileConfig } from 'utils/build/file-configs'; -import { Tester } from './tester'; import { killPidWithAllChilds } from 'utils/build/kill'; import * as utils from 'utils'; import fs from 'node:fs/promises'; @@ -17,28 +16,28 @@ import { // executes a command in background and returns a child process handle // by default pipes data to parent's stdio but this can be overridden export function background({ - command, - stdio = 'inherit', - cwd, - env - }: { + command, + stdio = 'inherit', + cwd, + env +}: { command: string; stdio: any; cwd?: ProcessEnvOptions['cwd']; env?: ProcessEnvOptions['env']; }): ChildProcessWithoutNullStreams { command = command.replace(/\n/g, ' '); - console.log(`Run command ${ command }`); - return _spawn(command, {stdio: stdio, shell: true, detached: true, cwd, env}); + console.log(`Run command ${command}`); + return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env }); } export function runInBackground({ - command, - components, - stdio, - cwd, - env - }: { + command, + components, + stdio, + cwd, + env +}: { command: string; components?: string[]; stdio: any; @@ -46,19 +45,19 @@ export function runInBackground({ env?: Parameters[0]['env']; }): ChildProcessWithoutNullStreams { if (components && components.length > 0) { - command += ` --components=${ components.join(',') }`; + command += ` --components=${components.join(',')}`; } - return background({command, stdio, cwd, env}); + return background({ command, stdio, cwd, env }); } export function runServerInBackground({ - components, - stdio, - cwd, - env, - useZkInception, - chain - }: { + components, + stdio, + cwd, + env, + useZkInception, + chain +}: { components?: string[]; stdio: any; cwd?: Parameters[0]['cwd']; @@ -72,12 +71,12 @@ export function runServerInBackground({ if (useZkInception) { command = 'zk_inception server'; if (chain) { - command += ` --chain ${ chain }`; + command += ` --chain ${chain}`; } } else { command = 'zk server'; } - return runInBackground({command, components, stdio, cwd, env}); + return runInBackground({ command, components, stdio, cwd, env }); } export interface MainNodeSpawnOptions { @@ -93,18 +92,13 @@ export enum NodeType { } export class Node { - constructor( - public readonly tester: Tester, - public proc: ChildProcessWithoutNullStreams, - private readonly type: TYPE - ) { - } + constructor(public proc: ChildProcessWithoutNullStreams, public l2NodeUrl: string, private readonly type: TYPE) {} public async terminate() { try { await killPidWithAllChilds(this.proc.pid!, 9); } catch (err) { - console.log(`ignored error: ${ err }`); + console.log(`ignored error: ${err}`); } } @@ -115,9 +109,9 @@ export class Node { */ public static async killAll(type: NodeType) { try { - await utils.exec(`killall -KILL ${ type }`); + await utils.exec(`killall -KILL ${type}`); } catch (err) { - console.log(`ignored error: ${ err }`); + console.log(`ignored error: ${err}`); } } @@ -135,7 +129,8 @@ export class Node { let iter = 0; while (iter < 30) { try { - await this.tester.syncWallet.provider.getBlockNumber(); + let provider = new zksync.Provider(this.l2NodeUrl); + await provider.getBlockNumber(); await utils.sleep(2); iter += 1; } catch (_) { @@ -144,7 +139,7 @@ export class Node { } } // It's going to panic anyway, since the server is a singleton entity, so better to exit early. - throw new Error(`${ this.type } didn't stop after a kill request`); + throw new Error(`${this.type} didn't stop after a kill request`); } } @@ -155,12 +150,11 @@ export class NodeSpawner { private readonly fileConfig: FileConfig, private readonly options: MainNodeSpawnOptions, private readonly env?: ProcessEnvOptions['env'] - ) { - } + ) {} public async spawnMainNode(newL1GasPrice?: string, newPubdataPrice?: string): Promise> { const env = this.env ?? process.env; - const {fileConfig, pathToHome, options, logs} = this; + const { fileConfig, pathToHome, options, logs } = this; const testMode = newPubdataPrice || newL1GasPrice; @@ -216,27 +210,25 @@ export class NodeSpawner { }); // Wait until the main node starts responding. - const tester = await Tester.init( - options.ethClientWeb3Url, - options.apiWeb3JsonRpcHttpUrl, - options.baseTokenAddress - ); - await waitForNodeToStart(tester, proc, options.apiWeb3JsonRpcHttpUrl); - return new Node(tester, proc, NodeType.MAIN); + await waitForNodeToStart(proc, options.apiWeb3JsonRpcHttpUrl); + return new Node(proc, options.apiWeb3JsonRpcHttpUrl, NodeType.MAIN); } } -async function waitForNodeToStart(tester: Tester, proc: ChildProcessWithoutNullStreams, l2Url: string) { +async function waitForNodeToStart(proc: ChildProcessWithoutNullStreams, l2Url: string) { while (true) { try { - const blockNumber = await tester.syncWallet.provider.getBlockNumber(); - console.log(`Initialized node API on ${ l2Url }; latest block: ${ blockNumber }`); - break; + const l2Provider = new zksync.Provider(l2Url); + const blockNumber = await l2Provider.getBlockNumber(); + if (blockNumber != 0) { + console.log(`Initialized node API on ${l2Url}; latest block: ${blockNumber}`); + break; + } } catch (err) { if (proc.exitCode != null) { - assert.fail(`server failed to start, exitCode = ${ proc.exitCode }`); + assert.fail(`server failed to start, exitCode = ${proc.exitCode}`); } - console.log(`Node waiting for API on ${ l2Url }`); + console.log(`Node waiting for API on ${l2Url}`); await utils.sleep(1); } } diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index dad3c23558b8..1140b4604b66 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -31,22 +31,22 @@ const MAX_GAS_PER_PUBDATA = 50_000n; // For CI we use only 2 gas prices to not slow it down too much. const L1_GAS_PRICES_TO_TEST = process.env.CI ? [ - 5_000_000_000n, // 5 gwei - 10_000_000_000n // 10 gwei - ] + 5_000_000_000n, // 5 gwei + 10_000_000_000n // 10 gwei + ] : [ - 1_000_000_000n, // 1 gwei - 5_000_000_000n, // 5 gwei - 10_000_000_000n, // 10 gwei - 25_000_000_000n, // 25 gwei - 50_000_000_000n, // 50 gwei - 100_000_000_000n, // 100 gwei - 200_000_000_000n, // 200 gwei - 400_000_000_000n, // 400 gwei - 800_000_000_000n, // 800 gwei - 1_000_000_000_000n, // 1000 gwei - 2_000_000_000_000n // 2000 gwei - ]; + 1_000_000_000n, // 1 gwei + 5_000_000_000n, // 5 gwei + 10_000_000_000n, // 10 gwei + 25_000_000_000n, // 25 gwei + 50_000_000_000n, // 50 gwei + 100_000_000_000n, // 100 gwei + 200_000_000_000n, // 200 gwei + 400_000_000_000n, // 400 gwei + 800_000_000_000n, // 800 gwei + 1_000_000_000_000n, // 1000 gwei + 2_000_000_000_000n // 2000 gwei + ]; // Unless `RUN_FEE_TEST` is provided, skip the test suit const testFees = process.env.RUN_FEE_TEST ? describe : describe.skip; @@ -75,16 +75,10 @@ testFees('Test fees', function () { beforeAll(async () => { testMaster = TestMaster.getInstance(__filename); - - // try { - // await killPidWithAllChilds(testMaster.environment().l2NodePid, 9); - // } catch (err) { - // console.log(`ignored error: ${ err }`); - // } - - // TODO: imporove the waiting logic - // console.log('Waiting for the server to stop...'); - // await utils.sleep(10); + let l2Node = testMaster.environment().l2Node; + if (l2Node !== undefined) { + await killPidWithAllChilds(l2Node.proc.pid!, 9); + } if (!fileConfig.loadFromFile) { ethClientWeb3Url = process.env.ETH_CLIENT_WEB3_URL!; @@ -114,7 +108,7 @@ testFees('Test fees', function () { const pathToMainLogs = await logsPath('server.log'); mainLogs = await fs.open(pathToMainLogs, 'a'); - console.log(`Writing server logs to ${ pathToMainLogs }`); + console.log(`Writing server logs to ${pathToMainLogs}`); mainNodeSpawner = new NodeSpawner(pathToHome, mainLogs, fileConfig, { enableConsensus, @@ -131,165 +125,163 @@ testFees('Test fees', function () { }); test('Test all fees', async () => { - // const receiver = ethers.Wallet.createRandom().address; - // - // // Getting ETH price in gas. - // const feeTestL1Receipt = await ( - // await alice.ethWallet().sendTransaction({ - // to: receiver, - // value: 1n - // }) - // ).wait(); - // - // if (feeTestL1Receipt === null) { - // throw new Error('Failed to send ETH transaction'); - // } - // - // const feeTestL1ReceiptERC20 = await ( - // await alice.ethWallet().sendTransaction({ - // to: aliceErc20.getAddress(), - // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]) - // }) - // ).wait(); - // - // if (feeTestL1ReceiptERC20 === null) { - // throw new Error('Failed to send ERC20 transaction'); - // } - // - // // Warming up slots for the receiver - // await ( - // await alice.sendTransaction({ - // to: receiver, - // value: BigInt(1) - // }) - // ).wait(); - // - // await ( - // await alice.sendTransaction({ - // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), - // to: tokenDetails.l2Address - // }) - // ).wait(); - // - // let reports = [ - // 'ETH transfer (to new):\n\n', - // 'ETH transfer (to old):\n\n', - // 'ERC20 transfer (to new):\n\n', - // 'ERC20 transfer (to old):\n\n' - // ]; - // for (const gasPrice of L1_GAS_PRICES_TO_TEST) { - // // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. - // await mainNode.killAndWaitForShutdown(); - // mainNode = await mainNodeSpawner.spawnMainNode(gasPrice.toString(), gasPrice.toString()); - // - // reports = await appendResults( - // alice, - // [feeTestL1Receipt, feeTestL1Receipt, feeTestL1ReceiptERC20, feeTestL1ReceiptERC20], - // // We always regenerate new addresses for transaction requests in order to estimate the cost for a new account - // [ - // { - // to: ethers.Wallet.createRandom().address, - // value: 1n - // }, - // { - // to: receiver, - // value: 1n - // }, - // { - // data: aliceErc20.interface.encodeFunctionData('transfer', [ - // ethers.Wallet.createRandom().address, - // 1n - // ]), - // to: tokenDetails.l2Address - // }, - // { - // data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), - // to: tokenDetails.l2Address - // } - // ], - // gasPrice, - // reports - // ); - // } - // - // console.log(`Full report: \n\n${ reports.join('\n\n') }`); + const receiver = ethers.Wallet.createRandom().address; + + // Getting ETH price in gas. + const feeTestL1Receipt = await ( + await alice.ethWallet().sendTransaction({ + to: receiver, + value: 1n + }) + ).wait(); + + if (feeTestL1Receipt === null) { + throw new Error('Failed to send ETH transaction'); + } + + const feeTestL1ReceiptERC20 = await ( + await alice.ethWallet().sendTransaction({ + to: aliceErc20.getAddress(), + data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]) + }) + ).wait(); + + if (feeTestL1ReceiptERC20 === null) { + throw new Error('Failed to send ERC20 transaction'); + } + + // Warming up slots for the receiver + await ( + await alice.sendTransaction({ + to: receiver, + value: BigInt(1) + }) + ).wait(); + + await ( + await alice.sendTransaction({ + data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), + to: tokenDetails.l2Address + }) + ).wait(); + + let reports = [ + 'ETH transfer (to new):\n\n', + 'ETH transfer (to old):\n\n', + 'ERC20 transfer (to new):\n\n', + 'ERC20 transfer (to old):\n\n' + ]; + for (const gasPrice of L1_GAS_PRICES_TO_TEST) { + // For the sake of simplicity, we'll use the same pubdata price as the L1 gas price. + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode(gasPrice.toString(), gasPrice.toString()); + + reports = await appendResults( + alice, + [feeTestL1Receipt, feeTestL1Receipt, feeTestL1ReceiptERC20, feeTestL1ReceiptERC20], + // We always regenerate new addresses for transaction requests in order to estimate the cost for a new account + [ + { + to: ethers.Wallet.createRandom().address, + value: 1n + }, + { + to: receiver, + value: 1n + }, + { + data: aliceErc20.interface.encodeFunctionData('transfer', [ + ethers.Wallet.createRandom().address, + 1n + ]), + to: tokenDetails.l2Address + }, + { + data: aliceErc20.interface.encodeFunctionData('transfer', [receiver, 1n]), + to: tokenDetails.l2Address + } + ], + gasPrice, + reports + ); + } + + console.log(`Full report: \n\n${reports.join('\n\n')}`); }); test('Test gas consumption under large L1 gas price', async () => { - // if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { - // // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. - // return; - // } - // - // // In this test we check that the server works fine when the required gasLimit is over u32::MAX. - // // Under normal server behavior, the maximal gas spent on pubdata is around 120kb * 2^20 gas/byte = ~120 * 10^9 gas. - // - // // In this test we will set gas per pubdata byte to its maximum value, while publishing a large L1->L2 message. - // - // const minimalL2GasPrice = testMaster.environment().minimalL2GasPrice; - // - // // We want the total gas limit to be over u32::MAX, so we need the gas per pubdata to be 50k. - // // - // // Note, that in case, any sort of overhead is present in the l2 fair gas price calculation, the final - // // gas per pubdata may be lower than 50_000. Here we assume that it is not the case, but we'll double check - // // that the gasLimit is indeed over u32::MAX, which is the most important tested property. - // const requiredPubdataPrice = minimalL2GasPrice * 100_000n; - // - // await mainNode.killAndWaitForShutdown(); - // mainNode = await mainNodeSpawner.spawnMainNode( - // requiredPubdataPrice.toString(), - // requiredPubdataPrice.toString() - // ); - // - // const l1Messenger = new ethers.Contract(zksync.utils.L1_MESSENGER_ADDRESS, zksync.utils.L1_MESSENGER, alice); - // - // // Firstly, let's test a successful transaction. - // const largeData = ethers.randomBytes(90_000); - // const tx = await l1Messenger.sendToL1(largeData, {type: 0}); - // expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); - // const receipt = await tx.wait(); - // expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); - // - // // Let's also check that the same transaction would work as eth_call - // const systemContextArtifact = getTestContract('ISystemContext'); - // const systemContext = new ethers.Contract(SYSTEM_CONTEXT_ADDRESS, systemContextArtifact.abi, alice.provider); - // const systemContextGasPerPubdataByte = await systemContext.gasPerPubdataByte(); - // expect(systemContextGasPerPubdataByte).toEqual(MAX_GAS_PER_PUBDATA); - // - // const dataHash = await l1Messenger.sendToL1.staticCall(largeData, {type: 0}); - // expect(dataHash).toEqual(ethers.keccak256(largeData)); - // - // // Secondly, let's test an unsuccessful transaction with large refund. - // - // // The size of the data has increased, so the previous gas limit is not enough. - // const largerData = ethers.randomBytes(91_000); - // const gasToPass = receipt.gasUsed; - // const unsuccessfulTx = await l1Messenger.sendToL1(largerData, { - // gasLimit: gasToPass, - // type: 0 - // }); + if (testMaster.environment().l1BatchCommitDataGeneratorMode === DataAvailabityMode.Validium) { + // We skip this test for Validium mode, since L1 gas price has little impact on the gasLimit in this mode. + return; + } + + // In this test we check that the server works fine when the required gasLimit is over u32::MAX. + // Under normal server behavior, the maximal gas spent on pubdata is around 120kb * 2^20 gas/byte = ~120 * 10^9 gas. + + // In this test we will set gas per pubdata byte to its maximum value, while publishing a large L1->L2 message. + + const minimalL2GasPrice = testMaster.environment().minimalL2GasPrice; + + // We want the total gas limit to be over u32::MAX, so we need the gas per pubdata to be 50k. // - // try { - // await unsuccessfulTx.wait(); - // throw new Error('The transaction should have reverted'); - // } catch { - // const receipt = await alice.provider.getTransactionReceipt(unsuccessfulTx.hash); - // expect(gasToPass - receipt!.gasUsed > UINT32_MAX).toBeTruthy(); - // } + // Note, that in case, any sort of overhead is present in the l2 fair gas price calculation, the final + // gas per pubdata may be lower than 50_000. Here we assume that it is not the case, but we'll double check + // that the gasLimit is indeed over u32::MAX, which is the most important tested property. + const requiredPubdataPrice = minimalL2GasPrice * 100_000n; + + await mainNode.killAndWaitForShutdown(); + mainNode = await mainNodeSpawner.spawnMainNode( + requiredPubdataPrice.toString(), + requiredPubdataPrice.toString() + ); + + const l1Messenger = new ethers.Contract(zksync.utils.L1_MESSENGER_ADDRESS, zksync.utils.L1_MESSENGER, alice); + + // Firstly, let's test a successful transaction. + const largeData = ethers.randomBytes(90_000); + const tx = await l1Messenger.sendToL1(largeData, { type: 0 }); + expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); + const receipt = await tx.wait(); + expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); + + // Let's also check that the same transaction would work as eth_call + const systemContextArtifact = getTestContract('ISystemContext'); + const systemContext = new ethers.Contract(SYSTEM_CONTEXT_ADDRESS, systemContextArtifact.abi, alice.provider); + const systemContextGasPerPubdataByte = await systemContext.gasPerPubdataByte(); + expect(systemContextGasPerPubdataByte).toEqual(MAX_GAS_PER_PUBDATA); + + const dataHash = await l1Messenger.sendToL1.staticCall(largeData, { type: 0 }); + expect(dataHash).toEqual(ethers.keccak256(largeData)); + + // Secondly, let's test an unsuccessful transaction with large refund. + + // The size of the data has increased, so the previous gas limit is not enough. + const largerData = ethers.randomBytes(91_000); + const gasToPass = receipt.gasUsed; + const unsuccessfulTx = await l1Messenger.sendToL1(largerData, { + gasLimit: gasToPass, + type: 0 + }); + + try { + await unsuccessfulTx.wait(); + throw new Error('The transaction should have reverted'); + } catch { + const receipt = await alice.provider.getTransactionReceipt(unsuccessfulTx.hash); + expect(gasToPass - receipt!.gasUsed > UINT32_MAX).toBeTruthy(); + } }); afterAll(async () => { - // Returning the pubdata price to the default one + await testMaster.deinitialize(); await mainNode.killAndWaitForShutdown(); - mainNode = await mainNodeSpawner.spawnMainNode(); - mainNode.proc.unref(); + // Returning the pubdata price to the default one // Restore defaults setTransactionSlots(pathToHome, fileConfig, 8192); deleteInternalEnforcedL1GasPrice(pathToHome, fileConfig); deleteInternalEnforcedPubdataPrice(pathToHome, fileConfig); - - await testMaster.deinitialize(); + mainNode = await mainNodeSpawner.spawnMainNode(); }); }); @@ -332,7 +324,7 @@ async function updateReport( const balanceBefore = await sender.getBalance(); const transaction = await sender.sendTransaction(transactionRequest); - console.log(`Sending transaction: ${ transaction.hash }`); + console.log(`Sending transaction: ${transaction.hash}`); await transaction.wait(); const balanceAfter = await sender.getBalance(); const balanceDiff = balanceBefore - balanceAfter; @@ -340,12 +332,12 @@ async function updateReport( const l2PriceAsNumber = +ethers.formatEther(balanceDiff); const l2EstimatedPriceAsNumber = +ethers.formatEther(estimatedPrice); - const gasReport = `Gas price ${ newL1GasPrice / 1000000000n } gwei: - L1 cost ${ expectedL1Price }, - L2 estimated cost: ${ l2EstimatedPriceAsNumber } - Estimated Gain: ${ expectedL1Price / l2EstimatedPriceAsNumber } - L2 cost: ${ l2PriceAsNumber }, - Gain: ${ expectedL1Price / l2PriceAsNumber }\n`; + const gasReport = `Gas price ${newL1GasPrice / 1000000000n} gwei: + L1 cost ${expectedL1Price}, + L2 estimated cost: ${l2EstimatedPriceAsNumber} + Estimated Gain: ${expectedL1Price / l2EstimatedPriceAsNumber} + L2 cost: ${l2PriceAsNumber}, + Gain: ${expectedL1Price / l2PriceAsNumber}\n`; console.log(gasReport); return oldReport + gasReport; diff --git a/core/tests/ts-integration/tests/utils.ts b/core/tests/ts-integration/tests/utils.ts index ead050437c61..24df8a170c20 100644 --- a/core/tests/ts-integration/tests/utils.ts +++ b/core/tests/ts-integration/tests/utils.ts @@ -30,8 +30,8 @@ function setPropertyInGeneralConfig(pathToHome: string, fileConfig: any, propert }); const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); - const regex = new RegExp(`${ property }:\\s*\\d+(\\.\\d+)?`, 'g'); - const newGeneralConfig = generalConfig.replace(regex, `${ property }: ${ value }`); + const regex = new RegExp(`${property}:\\s*\\d+(\\.\\d+)?`, 'g'); + const newGeneralConfig = generalConfig.replace(regex, `${property}: ${value}`); fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); } @@ -46,17 +46,17 @@ function setGasAdjusterProperty(pathToHome: string, fileConfig: any, property: s const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); // Define the regex pattern to check if the property already exists - const propertyRegex = new RegExp(`(^\\s*${ property }:\\s*\\d+(\\.\\d+)?$)`, 'm'); + const propertyRegex = new RegExp(`(^\\s*${property}:\\s*\\d+(\\.\\d+)?$)`, 'm'); const gasAdjusterRegex = new RegExp('(^\\s*gas_adjuster:.*$)', 'gm'); let newGeneralConfig; if (propertyRegex.test(generalConfig)) { // If the property exists, modify its value - newGeneralConfig = generalConfig.replace(propertyRegex, ` ${ property }: ${ value }`); + newGeneralConfig = generalConfig.replace(propertyRegex, ` ${property}: ${value}`); } else { // If the property does not exist, add it under the gas_adjuster section - newGeneralConfig = generalConfig.replace(gasAdjusterRegex, `$1\n ${ property }: ${ value }`); + newGeneralConfig = generalConfig.replace(gasAdjusterRegex, `$1\n ${property}: ${value}`); } fs.writeFileSync(generalConfigPath, newGeneralConfig, 'utf8'); @@ -72,7 +72,7 @@ function deleteProperty(pathToHome: string, fileConfig: any, property: string) { const generalConfig = fs.readFileSync(generalConfigPath, 'utf8'); // Define the regex pattern to find the property line and remove it completely - const propertyRegex = new RegExp(`^\\s*${ property }:.*\\n?`, 'm'); + const propertyRegex = new RegExp(`^\\s*${property}:.*\\n?`, 'm'); // Remove the line if the property exists const newGeneralConfig = generalConfig.replace(propertyRegex, ''); diff --git a/etc/utils/src/kill.ts b/etc/utils/src/kill.ts index a0932c1acf62..c718aabb4ebe 100644 --- a/etc/utils/src/kill.ts +++ b/etc/utils/src/kill.ts @@ -7,19 +7,19 @@ export async function killPidWithAllChilds(pid: number, signalNumber: number) { while (true) { try { let child = childs.at(-1); - childs.push(+(await promisify(exec)(`pgrep -P ${ child }`)).stdout); + childs.push(+(await promisify(exec)(`pgrep -P ${child}`)).stdout); } catch (e) { break; } } // We always run the test using additional tools, that means we have to kill not the main process, but the child process - console.log("childs", childs); + console.log('childs', childs); for (let i = childs.length - 1; i >= 0; i--) { - console.log(`kill ${ childs[i] }`); + console.log(`kill ${childs[i]}`); try { - await promisify(exec)(`kill -${ signalNumber } ${ childs[i] }`); + await promisify(exec)(`kill -${signalNumber} ${childs[i]}`); } catch (e) { - console.log(`Failed to kill ${ childs[i] } with ${ e }`,) + console.log(`Failed to kill ${childs[i]} with ${e}`); } } } diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs index c6a5db8dd292..1337566e5369 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/args/fees.rs @@ -1,10 +1,12 @@ use clap::Parser; use serde::{Deserialize, Serialize}; -use crate::messages::MSG_NO_DEPS_HELP; +use crate::messages::{MSG_NO_DEPS_HELP, MSG_NO_KILL_HELP}; #[derive(Debug, Serialize, Deserialize, Parser)] pub struct FeesArgs { #[clap(short, long, help = MSG_NO_DEPS_HELP)] pub no_deps: bool, + #[clap(short, long, help = MSG_NO_KILL_HELP)] + pub no_kill: bool, } diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index 99f48aa33a09..f8e7fb9e89c9 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -23,8 +23,10 @@ pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { ecosystem_config.current_chain() )); - let mut command = cmd!(shell, "yarn jest fees.test.ts") + let mut command = cmd!(shell, "yarn jest fees.test.ts --testTimeout 120000") + .env("SPAWN_NODE", "1") .env("RUN_FEE_TEST", "1") + .env("NO_KILL", args.no_kill.to_string()) .env("CHAIN_NAME", ecosystem_config.current_chain()); if global_config().verbose { From df0a4516a05b80564b504ccb1ae6684258fa5628 Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 25 Sep 2024 18:59:55 +0200 Subject: [PATCH 44/51] Reset pid Signed-off-by: Danil --- core/tests/ts-integration/src/context-owner.ts | 8 ++++++++ core/tests/ts-integration/src/env.ts | 4 ++-- .../ts-integration/src/jest-setup/global-setup.ts | 6 ------ core/tests/ts-integration/src/types.ts | 2 +- core/tests/ts-integration/tests/fees.test.ts | 13 +++++++++---- etc/utils/src/kill.ts | 2 +- 6 files changed, 21 insertions(+), 14 deletions(-) diff --git a/core/tests/ts-integration/src/context-owner.ts b/core/tests/ts-integration/src/context-owner.ts index 71c8227af2c5..73c202211e3b 100644 --- a/core/tests/ts-integration/src/context-owner.ts +++ b/core/tests/ts-integration/src/context-owner.ts @@ -7,6 +7,7 @@ import { lookupPrerequisites } from './prerequisites'; import { Reporter } from './reporter'; import { scaledGasPrice } from './helpers'; import { RetryProvider } from './retry-provider'; +import { killPidWithAllChilds } from 'utils/build/kill'; // These amounts of ETH would be provided to each test suite through its "main" account. // It is assumed to be enough to run a set of "normal" transactions. @@ -624,6 +625,9 @@ export class TestContextOwner { // Then propagate the exception. throw error; } + if (this.env.l2NodePid !== undefined) { + await killPidWithAllChilds(this.env.l2NodePid, 9); + } } /** @@ -648,6 +652,10 @@ export class TestContextOwner { // into account. If the same wallet would be reused (e.g. on stage), it'll just have to // deposit less next time. } + + async setPid(newPid: number) { + this.env.l2NodePid = newPid; + } } /** diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index 18d9c084f78a..9afc06d1dc99 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -177,7 +177,7 @@ async function loadTestEnvironmentFromFile(fileConfig: FileConfig): Promise { network, mainWalletPK, l2NodeUrl, - l2Node: undefined, + l2NodePid: undefined, l1NodeUrl, wsL2NodeUrl, healthcheckPort, diff --git a/core/tests/ts-integration/src/jest-setup/global-setup.ts b/core/tests/ts-integration/src/jest-setup/global-setup.ts index d050ae6705dd..27a47a1e9332 100644 --- a/core/tests/ts-integration/src/jest-setup/global-setup.ts +++ b/core/tests/ts-integration/src/jest-setup/global-setup.ts @@ -1,11 +1,5 @@ -import { shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { bigIntReplacer } from '../helpers'; import { TestContextOwner, loadTestEnvironment } from '../index'; -import path from 'path'; -import { NodeSpawner, runServerInBackground } from '../utils'; -import { exec } from 'utils'; -import fs from 'node:fs/promises'; -import { logsTestPath } from 'utils/build/logs'; declare global { var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; diff --git a/core/tests/ts-integration/src/types.ts b/core/tests/ts-integration/src/types.ts index ffb94ca32545..0302252e37b1 100644 --- a/core/tests/ts-integration/src/types.ts +++ b/core/tests/ts-integration/src/types.ts @@ -60,7 +60,7 @@ export interface TestEnvironment { /* * L2 node PID */ - l2Node: Node | undefined; + l2NodePid: number | undefined; /** * Plaintext name of the L1 network name (i.e. `localhost` or `goerli`). */ diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 1140b4604b66..8e2056b1f2d6 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -11,12 +11,12 @@ */ import * as utils from 'utils'; import fs from 'node:fs/promises'; -import { TestMaster } from '../src'; +import { TestContext, TestContextOwner, TestMaster } from '../src'; import * as zksync from 'zksync-ethers'; import * as ethers from 'ethers'; import { DataAvailabityMode, Token } from '../src/types'; -import { SYSTEM_CONTEXT_ADDRESS, getTestContract } from '../src/helpers'; +import { SYSTEM_CONTEXT_ADDRESS, getTestContract, bigIntReviver, bigIntReplacer } from '../src/helpers'; import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { logsTestPath } from 'utils/build/logs'; import path from 'path'; @@ -24,6 +24,10 @@ import { NodeSpawner, Node, NodeType } from '../src/utils'; import { deleteInternalEnforcedL1GasPrice, deleteInternalEnforcedPubdataPrice, setTransactionSlots } from './utils'; import { killPidWithAllChilds } from 'utils/build/kill'; +declare global { + var __ZKSYNC_TEST_CONTEXT_OWNER__: TestContextOwner; +} + const UINT32_MAX = 2n ** 32n - 1n; const MAX_GAS_PER_PUBDATA = 50_000n; @@ -75,9 +79,9 @@ testFees('Test fees', function () { beforeAll(async () => { testMaster = TestMaster.getInstance(__filename); - let l2Node = testMaster.environment().l2Node; + let l2Node = testMaster.environment().l2NodePid; if (l2Node !== undefined) { - await killPidWithAllChilds(l2Node.proc.pid!, 9); + await killPidWithAllChilds(l2Node, 9); } if (!fileConfig.loadFromFile) { @@ -282,6 +286,7 @@ testFees('Test fees', function () { deleteInternalEnforcedL1GasPrice(pathToHome, fileConfig); deleteInternalEnforcedPubdataPrice(pathToHome, fileConfig); mainNode = await mainNodeSpawner.spawnMainNode(); + await __ZKSYNC_TEST_CONTEXT_OWNER__.setPid(mainNode.proc.pid!); }); }); diff --git a/etc/utils/src/kill.ts b/etc/utils/src/kill.ts index c718aabb4ebe..6d0fcc507631 100644 --- a/etc/utils/src/kill.ts +++ b/etc/utils/src/kill.ts @@ -3,7 +3,7 @@ import { exec } from 'node:child_process'; export async function killPidWithAllChilds(pid: number, signalNumber: number) { let childs = [pid]; - console.log(pid, signalNumber); + console.log('kill child', pid, signalNumber); while (true) { try { let child = childs.at(-1); From 74e29510c45466e0e28008ac720d267b3ead844a Mon Sep 17 00:00:00 2001 From: Danil Date: Thu, 26 Sep 2024 10:28:18 +0200 Subject: [PATCH 45/51] Fix lints and increase timings Signed-off-by: Danil --- core/tests/ts-integration/src/env.ts | 1 - core/tests/ts-integration/src/types.ts | 2 -- core/tests/ts-integration/tests/fees.test.ts | 5 ++--- zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs | 2 +- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index 9afc06d1dc99..1de917c2362c 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -282,7 +282,6 @@ export async function loadTestEnvironmentFromEnv(): Promise { ); const healthcheckPort = process.env.API_HEALTHCHECK_PORT ?? '3071'; - const l2NodePid = 0; return { maxLogsLimit, pathToHome, diff --git a/core/tests/ts-integration/src/types.ts b/core/tests/ts-integration/src/types.ts index 0302252e37b1..c513480c1b41 100644 --- a/core/tests/ts-integration/src/types.ts +++ b/core/tests/ts-integration/src/types.ts @@ -1,5 +1,3 @@ -import { Node, NodeType } from './utils'; - export enum NodeMode { Main, External diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 8e2056b1f2d6..3b34468706f1 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -9,14 +9,13 @@ * sure that the test is maintained does not get broken. * */ -import * as utils from 'utils'; import fs from 'node:fs/promises'; -import { TestContext, TestContextOwner, TestMaster } from '../src'; +import { TestContextOwner, TestMaster } from '../src'; import * as zksync from 'zksync-ethers'; import * as ethers from 'ethers'; import { DataAvailabityMode, Token } from '../src/types'; -import { SYSTEM_CONTEXT_ADDRESS, getTestContract, bigIntReviver, bigIntReplacer } from '../src/helpers'; +import { SYSTEM_CONTEXT_ADDRESS, getTestContract } from '../src/helpers'; import { loadConfig, shouldLoadConfigFromFile } from 'utils/build/file-configs'; import { logsTestPath } from 'utils/build/logs'; import path from 'path'; diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index f8e7fb9e89c9..bdb788d1fd44 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -23,7 +23,7 @@ pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { ecosystem_config.current_chain() )); - let mut command = cmd!(shell, "yarn jest fees.test.ts --testTimeout 120000") + let mut command = cmd!(shell, "yarn jest fees.test.ts --testTimeout 240000") .env("SPAWN_NODE", "1") .env("RUN_FEE_TEST", "1") .env("NO_KILL", args.no_kill.to_string()) From f02b261cbea5aa6cc898c1f26fbd0ddb3a8bd16c Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 27 Sep 2024 10:46:32 +0200 Subject: [PATCH 46/51] Fix nits Signed-off-by: Danil --- .../tests/ts-integration/src/context-owner.ts | 4 +- core/tests/ts-integration/src/utils.ts | 56 +++---------------- core/tests/ts-integration/tests/fees.test.ts | 2 +- etc/utils/src/kill.ts | 3 - 4 files changed, 13 insertions(+), 52 deletions(-) diff --git a/core/tests/ts-integration/src/context-owner.ts b/core/tests/ts-integration/src/context-owner.ts index 73c202211e3b..e77cdf1a0531 100644 --- a/core/tests/ts-integration/src/context-owner.ts +++ b/core/tests/ts-integration/src/context-owner.ts @@ -626,7 +626,9 @@ export class TestContextOwner { throw error; } if (this.env.l2NodePid !== undefined) { + this.reporter.startAction(`Terminating L2 node process`); await killPidWithAllChilds(this.env.l2NodePid, 9); + this.reporter.finishAction(); } } @@ -653,7 +655,7 @@ export class TestContextOwner { // deposit less next time. } - async setPid(newPid: number) { + setL2NodePid(newPid: number) { this.env.l2NodePid = newPid; } } diff --git a/core/tests/ts-integration/src/utils.ts b/core/tests/ts-integration/src/utils.ts index 9d841792dbaf..128d0be57d00 100644 --- a/core/tests/ts-integration/src/utils.ts +++ b/core/tests/ts-integration/src/utils.ts @@ -15,41 +15,6 @@ import { // executes a command in background and returns a child process handle // by default pipes data to parent's stdio but this can be overridden -export function background({ - command, - stdio = 'inherit', - cwd, - env -}: { - command: string; - stdio: any; - cwd?: ProcessEnvOptions['cwd']; - env?: ProcessEnvOptions['env']; -}): ChildProcessWithoutNullStreams { - command = command.replace(/\n/g, ' '); - console.log(`Run command ${command}`); - return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env }); -} - -export function runInBackground({ - command, - components, - stdio, - cwd, - env -}: { - command: string; - components?: string[]; - stdio: any; - cwd?: Parameters[0]['cwd']; - env?: Parameters[0]['env']; -}): ChildProcessWithoutNullStreams { - if (components && components.length > 0) { - command += ` --components=${components.join(',')}`; - } - return background({ command, stdio, cwd, env }); -} - export function runServerInBackground({ components, stdio, @@ -60,8 +25,8 @@ export function runServerInBackground({ }: { components?: string[]; stdio: any; - cwd?: Parameters[0]['cwd']; - env?: Parameters[0]['env']; + cwd?: ProcessEnvOptions['cwd']; + env?: ProcessEnvOptions['env']; useZkInception?: boolean; newL1GasPrice?: string; newPubdataPrice?: string; @@ -76,7 +41,12 @@ export function runServerInBackground({ } else { command = 'zk server'; } - return runInBackground({ command, components, stdio, cwd, env }); + if (components && components.length > 0) { + command += ` --components=${components.join(',')}`; + } + command = command.replace(/\n/g, ' '); + console.log(`Run command ${command}`); + return _spawn(command, { stdio: stdio, shell: true, detached: true, cwd, env }); } export interface MainNodeSpawnOptions { @@ -115,14 +85,6 @@ export class Node { } } - /** Waits for the node process to exit. */ - public async waitForExit(): Promise { - while (this.proc.exitCode === null) { - await utils.sleep(1); - } - return this.proc.exitCode; - } - public async killAndWaitForShutdown() { await this.terminate(); // Wait until it's really stopped. @@ -149,7 +111,7 @@ export class NodeSpawner { private readonly logs: fs.FileHandle, private readonly fileConfig: FileConfig, private readonly options: MainNodeSpawnOptions, - private readonly env?: ProcessEnvOptions['env'] + private env?: ProcessEnvOptions['env'] ) {} public async spawnMainNode(newL1GasPrice?: string, newPubdataPrice?: string): Promise> { diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 3b34468706f1..b3e4c2862bcf 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -285,7 +285,7 @@ testFees('Test fees', function () { deleteInternalEnforcedL1GasPrice(pathToHome, fileConfig); deleteInternalEnforcedPubdataPrice(pathToHome, fileConfig); mainNode = await mainNodeSpawner.spawnMainNode(); - await __ZKSYNC_TEST_CONTEXT_OWNER__.setPid(mainNode.proc.pid!); + __ZKSYNC_TEST_CONTEXT_OWNER__.setL2NodePid(mainNode.proc.pid!); }); }); diff --git a/etc/utils/src/kill.ts b/etc/utils/src/kill.ts index 6d0fcc507631..71b76e71d805 100644 --- a/etc/utils/src/kill.ts +++ b/etc/utils/src/kill.ts @@ -3,7 +3,6 @@ import { exec } from 'node:child_process'; export async function killPidWithAllChilds(pid: number, signalNumber: number) { let childs = [pid]; - console.log('kill child', pid, signalNumber); while (true) { try { let child = childs.at(-1); @@ -13,9 +12,7 @@ export async function killPidWithAllChilds(pid: number, signalNumber: number) { } } // We always run the test using additional tools, that means we have to kill not the main process, but the child process - console.log('childs', childs); for (let i = childs.length - 1; i >= 0; i--) { - console.log(`kill ${childs[i]}`); try { await promisify(exec)(`kill -${signalNumber} ${childs[i]}`); } catch (e) { From fb31b2d901b5876d7d6c98f9b1f0fb1954e5f016 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 27 Sep 2024 16:16:47 +0200 Subject: [PATCH 47/51] Fix logs place Signed-off-by: Danil --- core/tests/ts-integration/tests/fees.test.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index b3e4c2862bcf..2297d7232ccb 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -72,8 +72,9 @@ testFees('Test fees', function () { const pathToHome = path.join(__dirname, '../../../..'); const enableConsensus = process.env.ENABLE_CONSENSUS == 'true'; - async function logsPath(name: string): Promise { - return await logsTestPath(fileConfig.chain, 'logs/server/fees', name); + async function logsPath(chain: string | undefined, name: string): Promise { + chain = chain ? chain : 'default'; + return await logsTestPath(chain, 'logs/server/fees', name); } beforeAll(async () => { @@ -109,7 +110,7 @@ testFees('Test fees', function () { baseTokenAddress = contractsConfig.l1.base_token_addr; } - const pathToMainLogs = await logsPath('server.log'); + const pathToMainLogs = await logsPath(fileConfig.chain, 'server.log'); mainLogs = await fs.open(pathToMainLogs, 'a'); console.log(`Writing server logs to ${pathToMainLogs}`); From 1cec206a9757e2d755b94a039409d6d5378ca326 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 27 Sep 2024 16:30:13 +0200 Subject: [PATCH 48/51] Run all fee tests Signed-off-by: Danil --- .github/workflows/ci-core-reusable.yml | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 464eef067680..6d22590b65bc 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -460,10 +460,23 @@ jobs: - name: Fee projection tests run: | ci_run killall -INT zksync_server || true - ci_run zk_supervisor test fees --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log - ci_run zk_supervisor test fees --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log - ci_run zk_supervisor test fees --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log - ci_run zk_supervisor test fees --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log + + ci_run zk_supervisor test fees --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log & + PID1=$! + + ci_run zk_supervisor test fees --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log & + PID2=$! + + ci_run zk_supervisor test fees --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log & + PID3=$! + + ci_run zk_supervisor test fees --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log & + PID4=$! + + wait $PID1 + wait $PID2 + wait $PID3 + wait $PID4 - name: Run revert tests run: | From 77e8814f48303f2edae4b023f5be98059fd02f42 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 27 Sep 2024 17:12:46 +0200 Subject: [PATCH 49/51] Fix multiple chains for fees Signed-off-by: Danil --- .../zk_supervisor/src/commands/test/fees.rs | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs index bdb788d1fd44..4e69c0efb971 100644 --- a/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs +++ b/zk_toolbox/crates/zk_supervisor/src/commands/test/fees.rs @@ -1,3 +1,6 @@ +use std::path::PathBuf; + +use anyhow::Context; use common::{cmd::Cmd, config::global_config, logger}; use config::EcosystemConfig; use xshell::{cmd, Shell}; @@ -6,11 +9,20 @@ use super::{ args::fees::FeesArgs, utils::{build_contracts, install_and_build_dependencies, TS_INTEGRATION_PATH}, }; -use crate::messages::MSG_INTEGRATION_TESTS_RUN_SUCCESS; +use crate::{ + commands::test::utils::{TestWallets, TEST_WALLETS_PATH}, + messages::{ + MSG_CHAIN_NOT_FOUND_ERR, MSG_DESERIALIZE_TEST_WALLETS_ERR, + MSG_INTEGRATION_TESTS_RUN_SUCCESS, + }, +}; pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; shell.change_dir(ecosystem_config.link_to_code.join(TS_INTEGRATION_PATH)); + let chain_config = ecosystem_config + .load_chain(global_config().chain_name.clone()) + .expect(MSG_CHAIN_NOT_FOUND_ERR); if !args.no_deps { logger::info("Installing dependencies"); @@ -23,11 +35,20 @@ pub async fn run(shell: &Shell, args: FeesArgs) -> anyhow::Result<()> { ecosystem_config.current_chain() )); + let wallets_path: PathBuf = ecosystem_config.link_to_code.join(TEST_WALLETS_PATH); + let wallets: TestWallets = serde_json::from_str(shell.read_file(&wallets_path)?.as_ref()) + .context(MSG_DESERIALIZE_TEST_WALLETS_ERR)?; + + wallets + .init_test_wallet(&ecosystem_config, &chain_config) + .await?; + let mut command = cmd!(shell, "yarn jest fees.test.ts --testTimeout 240000") .env("SPAWN_NODE", "1") .env("RUN_FEE_TEST", "1") .env("NO_KILL", args.no_kill.to_string()) - .env("CHAIN_NAME", ecosystem_config.current_chain()); + .env("CHAIN_NAME", ecosystem_config.current_chain()) + .env("MASTER_WALLET_PK", wallets.get_test_pk(&chain_config)?); if global_config().verbose { command = command.env( From 4028d35dadd1fcb576bd72ba955432b3d58ee435 Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 27 Sep 2024 17:47:38 +0200 Subject: [PATCH 50/51] Add no deps Signed-off-by: Danil --- .github/workflows/ci-core-reusable.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index e6d434e580b9..15d26e0b4691 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -465,16 +465,16 @@ jobs: run: | ci_run killall -INT zksync_server || true - ci_run zk_supervisor test fees --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log & + ci_run zk_supervisor test fees --no-deps --no-kill --chain era &> ${{ env.FEES_LOGS_DIR }}/era.log & PID1=$! - ci_run zk_supervisor test fees --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log & + ci_run zk_supervisor test fees --no-deps --no-kill --chain validium &> ${{ env.FEES_LOGS_DIR }}/validium.log & PID2=$! - ci_run zk_supervisor test fees --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log & + ci_run zk_supervisor test fees --no-deps --no-kill --chain custom_token &> ${{ env.FEES_LOGS_DIR }}/custom_token.log & PID3=$! - ci_run zk_supervisor test fees --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log & + ci_run zk_supervisor test fees --no-deps --no-kill --chain consensus &> ${{ env.FEES_LOGS_DIR }}/consensus.log & PID4=$! wait $PID1 From c89b3516898be3b5c9cc10767653b1b99882abd9 Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 30 Sep 2024 10:15:00 +0200 Subject: [PATCH 51/51] add logs Signed-off-by: Danil --- core/tests/ts-integration/tests/fees.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/core/tests/ts-integration/tests/fees.test.ts b/core/tests/ts-integration/tests/fees.test.ts index 2297d7232ccb..c41f5943efb5 100644 --- a/core/tests/ts-integration/tests/fees.test.ts +++ b/core/tests/ts-integration/tests/fees.test.ts @@ -246,6 +246,7 @@ testFees('Test fees', function () { const tx = await l1Messenger.sendToL1(largeData, { type: 0 }); expect(tx.gasLimit > UINT32_MAX).toBeTruthy(); const receipt = await tx.wait(); + console.log(`Gas used ${receipt.gasUsed}`); expect(receipt.gasUsed > UINT32_MAX).toBeTruthy(); // Let's also check that the same transaction would work as eth_call