From 8c9b3326297dfbcba55913182f865e13734d4b41 Mon Sep 17 00:00:00 2001 From: Artem Makhortov <13339874+artmakh@users.noreply.github.com> Date: Wed, 2 Oct 2024 22:39:45 +0700 Subject: [PATCH 1/8] fix(ci): Bring back hack for contracts build till full migration to foundry (#3000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Creation of empty dirs, to support building from both new (foundry-built) and old contracts ## Why ❔ ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`. --- .github/workflows/build-core-template.yml | 4 ++++ docker/Makefile | 4 +++- docker/external-node/Dockerfile | 2 ++ docker/server-v2/Dockerfile | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml index 9d00f98b1816..deaf087cd3eb 100644 --- a/.github/workflows/build-core-template.yml +++ b/.github/workflows/build-core-template.yml @@ -71,11 +71,15 @@ jobs: if [ $(jq length <<<"$tags") -eq 0 ]; then echo "No tag found on all pages." echo "BUILD_CONTRACTS=true" >> "$GITHUB_ENV" + # TODO Remove it when we migrate to foundry inside contracts repository + mkdir -p contracts/l1-contracts/artifacts/ exit 0 fi filtered_tag=$(jq -r --arg commit_sha "$commit_sha" 'map(select(.commit.sha == $commit_sha)) | .[].name' <<<"$tags") if [[ ! -z "$filtered_tag" ]]; then echo "BUILD_CONTRACTS=false" >> "$GITHUB_ENV" + # TODO Remove it when we migrate to foundry inside contracts repository + mkdir -p contracts/l1-contracts/out break fi ((page++)) diff --git a/docker/Makefile b/docker/Makefile index 72189902aa1a..444a94ce2214 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -61,12 +61,14 @@ check-contracts: fi # Build and download needed contracts +# TODO Remove mkdir once we use foundry inside contracts repo prepare-contracts: check-tools check-contracts @cd ../ && \ export ZKSYNC_HOME=$$(pwd) && \ export PATH=$$PATH:$${ZKSYNC_HOME}/bin && \ zkt || true && \ - zk_supervisor contracts + zk_supervisor contracts && \ + mkdir -p contracts/l1-contracts/artifacts # Download setup-key prepare-keys: diff --git a/docker/external-node/Dockerfile b/docker/external-node/Dockerfile index aa1089ae7b33..1012eecfc162 100644 --- a/docker/external-node/Dockerfile +++ b/docker/external-node/Dockerfile @@ -29,6 +29,8 @@ COPY contracts/system-contracts/contracts-preprocessed/precompiles/artifacts/ /c COPY contracts/system-contracts/artifacts-zk /contracts/system-contracts/artifacts-zk COPY contracts/l1-contracts/out/ /contracts/l1-contracts/out/ COPY contracts/l2-contracts/artifacts-zk/ /contracts/l2-contracts/artifacts-zk/ +# TODO Remove once we use foundry inside contracts repo +COPY contracts/l1-contracts/artifacts/ /contracts/l1-contracts/artifacts/ COPY etc/tokens/ /etc/tokens/ COPY etc/ERC20/ /etc/ERC20/ COPY etc/multivm_bootloaders/ /etc/multivm_bootloaders/ diff --git a/docker/server-v2/Dockerfile b/docker/server-v2/Dockerfile index 3e8b4f16bca1..13a391333270 100644 --- a/docker/server-v2/Dockerfile +++ b/docker/server-v2/Dockerfile @@ -37,6 +37,8 @@ COPY contracts/system-contracts/contracts-preprocessed/precompiles/artifacts/ /c COPY contracts/system-contracts/artifacts-zk /contracts/system-contracts/artifacts-zk COPY contracts/l1-contracts/out/ /contracts/l1-contracts/out/ COPY contracts/l2-contracts/artifacts-zk/ /contracts/l2-contracts/artifacts-zk/ +# TODO Remove once we use foundry inside contracts repo +COPY contracts/l1-contracts/artifacts/ /contracts/l1-contracts/artifacts/ COPY etc/tokens/ /etc/tokens/ COPY etc/ERC20/ /etc/ERC20/ COPY etc/multivm_bootloaders/ /etc/multivm_bootloaders/ From 8d24eb5f813efefd756906e41d8bbaaa3c92eb8f Mon Sep 17 00:00:00 2001 From: zksync-era-bot <147085853+zksync-era-bot@users.noreply.github.com> Date: Wed, 2 Oct 2024 21:07:14 +0400 Subject: [PATCH 2/8] chore(main): release core 24.28.0 (#2968) :robot: I have created a release *beep* *boop* --- ## [24.28.0](https://github.com/matter-labs/zksync-era/compare/core-v24.27.0...core-v24.28.0) (2024-10-02) ### Features * **da-clients:** add secrets ([#2954](https://github.com/matter-labs/zksync-era/issues/2954)) ([f4631e4](https://github.com/matter-labs/zksync-era/commit/f4631e4466de620cc1401b326d864cdb8b48a05d)) * **eth-sender:** add a cap to time_in_mempool ([#2978](https://github.com/matter-labs/zksync-era/issues/2978)) ([650d42f](https://github.com/matter-labs/zksync-era/commit/650d42fea6124d80b60a8270a303d72ad6ac741e)) * **eth-watch:** redesign to support multiple chains ([#2867](https://github.com/matter-labs/zksync-era/issues/2867)) ([aa72d84](https://github.com/matter-labs/zksync-era/commit/aa72d849c24a664acd083eba73795ddc5d31d55f)) * Expose http debug page ([#2952](https://github.com/matter-labs/zksync-era/issues/2952)) ([e0b6488](https://github.com/matter-labs/zksync-era/commit/e0b64888aae7324aec2d40fa0cd51ea7e1450cd9)) * **zk_toolbox:** add fees integration test to toolbox ([#2898](https://github.com/matter-labs/zksync-era/issues/2898)) ([e7ead76](https://github.com/matter-labs/zksync-era/commit/e7ead760ce0417dd36af3839ac557f7e9ab238a4)) * **zk_toolbox:** Add SQL format for zk supervisor ([#2950](https://github.com/matter-labs/zksync-era/issues/2950)) ([540e5d7](https://github.com/matter-labs/zksync-era/commit/540e5d7554f54e80d52f1bfae37e03ca8f787baf)) ### Bug Fixes * **api:** Fix batch fee input for `debug` namespace ([#2948](https://github.com/matter-labs/zksync-era/issues/2948)) ([79b6fcf](https://github.com/matter-labs/zksync-era/commit/79b6fcf8b5d10a0ccdceb846370dd6870b6a32b5)) * chainstack block limit exceeded ([#2974](https://github.com/matter-labs/zksync-era/issues/2974)) ([4ffbf42](https://github.com/matter-labs/zksync-era/commit/4ffbf426de166c11aaf5d7b5ed7d199644fba229)) * **eth-watch:** add missing check that from_block is not larger than finalized_block ([#2969](https://github.com/matter-labs/zksync-era/issues/2969)) ([3f406c7](https://github.com/matter-labs/zksync-era/commit/3f406c7d0c0e76d798c2d838abde57ca692822c0)) * ignore unknown fields in rpc json response ([#2962](https://github.com/matter-labs/zksync-era/issues/2962)) ([692ea73](https://github.com/matter-labs/zksync-era/commit/692ea73f75a5fb9db2b4ac33ad24d20568638742)) ### Performance Improvements * **api:** More efficient gas estimation ([#2937](https://github.com/matter-labs/zksync-era/issues/2937)) ([3b69e37](https://github.com/matter-labs/zksync-era/commit/3b69e37e470dab859a55787f6cc971e7083de2fd)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --------- Co-authored-by: zksync-era-bot --- .github/release-please/manifest.json | 2 +- Cargo.lock | 2 +- core/CHANGELOG.md | 25 +++++++++++++++++++++++++ core/bin/external_node/Cargo.toml | 2 +- 4 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.github/release-please/manifest.json b/.github/release-please/manifest.json index 44e10fb13fdf..e0e8fbeecf74 100644 --- a/.github/release-please/manifest.json +++ b/.github/release-please/manifest.json @@ -1,5 +1,5 @@ { - "core": "24.27.0", + "core": "24.28.0", "prover": "16.5.0", "zk_toolbox": "0.1.2" } diff --git a/Cargo.lock b/Cargo.lock index 127921ba3e9e..0873faae904c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10178,7 +10178,7 @@ dependencies = [ [[package]] name = "zksync_external_node" -version = "24.27.0" +version = "24.28.0" dependencies = [ "anyhow", "assert_matches", diff --git a/core/CHANGELOG.md b/core/CHANGELOG.md index 6cf2ff4419a9..b2f27a6630cd 100644 --- a/core/CHANGELOG.md +++ b/core/CHANGELOG.md @@ -1,5 +1,30 @@ # Changelog +## [24.28.0](https://github.com/matter-labs/zksync-era/compare/core-v24.27.0...core-v24.28.0) (2024-10-02) + + +### Features + +* **da-clients:** add secrets ([#2954](https://github.com/matter-labs/zksync-era/issues/2954)) ([f4631e4](https://github.com/matter-labs/zksync-era/commit/f4631e4466de620cc1401b326d864cdb8b48a05d)) +* **eth-sender:** add a cap to time_in_mempool ([#2978](https://github.com/matter-labs/zksync-era/issues/2978)) ([650d42f](https://github.com/matter-labs/zksync-era/commit/650d42fea6124d80b60a8270a303d72ad6ac741e)) +* **eth-watch:** redesign to support multiple chains ([#2867](https://github.com/matter-labs/zksync-era/issues/2867)) ([aa72d84](https://github.com/matter-labs/zksync-era/commit/aa72d849c24a664acd083eba73795ddc5d31d55f)) +* Expose http debug page ([#2952](https://github.com/matter-labs/zksync-era/issues/2952)) ([e0b6488](https://github.com/matter-labs/zksync-era/commit/e0b64888aae7324aec2d40fa0cd51ea7e1450cd9)) +* **zk_toolbox:** add fees integration test to toolbox ([#2898](https://github.com/matter-labs/zksync-era/issues/2898)) ([e7ead76](https://github.com/matter-labs/zksync-era/commit/e7ead760ce0417dd36af3839ac557f7e9ab238a4)) +* **zk_toolbox:** Add SQL format for zk supervisor ([#2950](https://github.com/matter-labs/zksync-era/issues/2950)) ([540e5d7](https://github.com/matter-labs/zksync-era/commit/540e5d7554f54e80d52f1bfae37e03ca8f787baf)) + + +### Bug Fixes + +* **api:** Fix batch fee input for `debug` namespace ([#2948](https://github.com/matter-labs/zksync-era/issues/2948)) ([79b6fcf](https://github.com/matter-labs/zksync-era/commit/79b6fcf8b5d10a0ccdceb846370dd6870b6a32b5)) +* chainstack block limit exceeded ([#2974](https://github.com/matter-labs/zksync-era/issues/2974)) ([4ffbf42](https://github.com/matter-labs/zksync-era/commit/4ffbf426de166c11aaf5d7b5ed7d199644fba229)) +* **eth-watch:** add missing check that from_block is not larger than finalized_block ([#2969](https://github.com/matter-labs/zksync-era/issues/2969)) ([3f406c7](https://github.com/matter-labs/zksync-era/commit/3f406c7d0c0e76d798c2d838abde57ca692822c0)) +* ignore unknown fields in rpc json response ([#2962](https://github.com/matter-labs/zksync-era/issues/2962)) ([692ea73](https://github.com/matter-labs/zksync-era/commit/692ea73f75a5fb9db2b4ac33ad24d20568638742)) + + +### Performance Improvements + +* **api:** More efficient gas estimation ([#2937](https://github.com/matter-labs/zksync-era/issues/2937)) ([3b69e37](https://github.com/matter-labs/zksync-era/commit/3b69e37e470dab859a55787f6cc971e7083de2fd)) + ## [24.27.0](https://github.com/matter-labs/zksync-era/compare/core-v24.26.0...core-v24.27.0) (2024-09-25) diff --git a/core/bin/external_node/Cargo.toml b/core/bin/external_node/Cargo.toml index d841ee5b42e6..086d381ecc30 100644 --- a/core/bin/external_node/Cargo.toml +++ b/core/bin/external_node/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "zksync_external_node" description = "Non-validator ZKsync node" -version = "24.27.0" # x-release-please-version +version = "24.28.0" # x-release-please-version edition.workspace = true authors.workspace = true homepage.workspace = true From 057705e55c3816d89478994d6f6c63b08a19156b Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Wed, 2 Oct 2024 12:38:54 -0600 Subject: [PATCH 3/8] feat(zk_toolbox): Add subcommands and flags for chain registration (#2946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Added subcommands: `zki chain init configs` - just creates configs with an intention to run chain initialization manually via subcommands `zki chain register-chain` - runs steps from `RegisterHyperchain.s.sol` `zki chain accept-chain-ownership` - accepts ownership for `DiamondProxy` `zki chain genesis database` - initializes database only, performs migration (uses values from args or `secrets.yaml`) `zki chain genesis server` - runs server --genesis Added flags: `zki ecosystem init --ecosystem-only` - runs `init` only for ecosystem (skips `init` for chain) Other changes: * Fixed issue with `--wallet_path` value ignored * Nullify database names if `zki ecosystem init` is used for multiple chains * Zeroify some addresses in `contracts.yaml` when copying from ecosystem during init ## Why ❔ These changes allow us to run the chain registration process for externally hosted chains. Not ideal yet, but the process goes like this: L1 side: * Init ecosystem: `zki ecosystem create && zki ecosystem init --ecosystem-only && zki chain init configs` * Fill in wallets * Register chain: `zki chain register-chain` * Deploy L2 contracts: `zki chain deploy-l2-contracts` * Share `contracts.yaml` L2 side: * Init ecosystem: `zki ecosystem create && zki ecosystem init --ecosystem-only && zki chain init configs` * Fill in wallets * Copy `contracts.yaml` * Accept ownership: `zki chain accept-chain-ownership` * Initialize databases: `zki chain genesis database` * Run server genesis: `zki chain genesis server` ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --- yarn.lock | 31 +---- .../commands/chain/accept_chain_ownership.rs | 42 ++++++ .../src/commands/chain/args/create.rs | 2 +- .../src/commands/chain/args/genesis.rs | 41 ++++++ .../src/commands/chain/args/init/configs.rs | 70 ++++++++++ .../chain/args/{init.rs => init/mod.rs} | 5 +- .../src/commands/chain/build_transactions.rs | 5 +- .../zk_inception/src/commands/chain/common.rs | 58 +------- .../chain/{genesis.rs => genesis/database.rs} | 129 +++++++----------- .../src/commands/chain/genesis/mod.rs | 94 +++++++++++++ .../src/commands/chain/genesis/server.rs | 46 +++++++ .../src/commands/chain/init/configs.rs | 107 +++++++++++++++ .../commands/chain/{init.rs => init/mod.rs} | 127 ++++++++--------- .../zk_inception/src/commands/chain/mod.rs | 42 ++++-- .../src/commands/chain/register_chain.rs | 96 +++++++++++++ .../src/commands/ecosystem/args/init.rs | 35 +++-- .../src/commands/ecosystem/init.rs | 102 ++++++++------ zk_toolbox/crates/zk_inception/src/main.rs | 8 +- .../crates/zk_inception/src/messages.rs | 11 +- 19 files changed, 732 insertions(+), 319 deletions(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/accept_chain_ownership.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/args/init/configs.rs rename zk_toolbox/crates/zk_inception/src/commands/chain/args/{init.rs => init/mod.rs} (96%) rename zk_toolbox/crates/zk_inception/src/commands/chain/{genesis.rs => genesis/database.rs} (63%) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/genesis/server.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs rename zk_toolbox/crates/zk_inception/src/commands/chain/{init.rs => init/mod.rs} (55%) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/chain/register_chain.rs diff --git a/yarn.lock b/yarn.lock index 3c764c7c7b7f..531f49abc000 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9816,7 +9816,7 @@ string-length@^4.0.1: char-regex "^1.0.2" strip-ansi "^6.0.0" -"string-width-cjs@npm:string-width@^4.2.0": +"string-width-cjs@npm:string-width@^4.2.0", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -9833,15 +9833,6 @@ string-width@^2.1.0, string-width@^2.1.1: is-fullwidth-code-point "^2.0.0" strip-ansi "^4.0.0" -string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: - version "4.2.3" - resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - string-width@^5.0.1, string-width@^5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/string-width/-/string-width-5.1.2.tgz#14f8daec6d81e7221d2a357e668cab73bdbca794" @@ -9908,7 +9899,7 @@ string_decoder@~1.1.1: dependencies: safe-buffer "~5.1.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -9929,13 +9920,6 @@ strip-ansi@^5.1.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1: version "7.1.0" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.1.0.tgz#d5b6568ca689d8561370b0707685d22434faff45" @@ -10781,16 +10765,7 @@ workerpool@6.2.1: resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343" integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw== -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": - version "7.0.0" - resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - -wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/accept_chain_ownership.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/accept_chain_ownership.rs new file mode 100644 index 000000000000..37d69fcf5bcc --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/accept_chain_ownership.rs @@ -0,0 +1,42 @@ +use anyhow::Context; +use common::{forge::ForgeScriptArgs, logger, spinner::Spinner}; +use config::EcosystemConfig; +use xshell::Shell; + +use crate::{ + accept_ownership::accept_admin, + messages::{ + MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_NOT_INITIALIZED, MSG_CHAIN_OWNERSHIP_TRANSFERRED, + MSG_L1_SECRETS_MUST_BE_PRESENTED, + }, +}; + +pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_current_chain() + .context(MSG_CHAIN_NOT_INITIALIZED)?; + let contracts = chain_config.get_contracts_config()?; + let secrets = chain_config.get_secrets_config()?; + let l1_rpc_url = secrets + .l1 + .context(MSG_L1_SECRETS_MUST_BE_PRESENTED)? + .l1_rpc_url + .expose_str() + .to_string(); + + let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); + accept_admin( + shell, + &ecosystem_config, + contracts.l1.chain_admin_addr, + chain_config.get_wallets_config()?.governor_private_key(), + contracts.l1.diamond_proxy_addr, + &args, + l1_rpc_url.clone(), + ) + .await?; + spinner.finish(); + logger::success(MSG_CHAIN_OWNERSHIP_TRANSFERRED); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/create.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/create.rs index 3ea15d10f8be..5fc46c1b227a 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/create.rs @@ -127,7 +127,7 @@ impl ChainCreateArgs { .ask() }); - let wallet_path: Option = if self.wallet_creation == Some(WalletCreation::InFile) { + let wallet_path: Option = if wallet_creation == WalletCreation::InFile { Some(self.wallet_path.unwrap_or_else(|| { Prompt::new(MSG_WALLET_PATH_PROMPT) .validate_with(|val: &String| { diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs index 483b78e9b267..21796b3179de 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs @@ -1,3 +1,4 @@ +use anyhow::Context; use clap::Parser; use common::{db::DatabaseConfig, Prompt}; use config::ChainConfig; @@ -77,6 +78,46 @@ impl GenesisArgs { } } } + + pub fn fill_values_with_secrets( + mut self, + chain_config: &ChainConfig, + ) -> anyhow::Result { + let secrets = chain_config.get_secrets_config()?; + let database = secrets + .database + .context("Database secrets must be present")?; + + let (server_db_url, server_db_name) = if let Some(db_full_url) = database.server_url { + let db_config = DatabaseConfig::from_url(db_full_url.expose_url()) + .context("Invalid server database URL")?; + (Some(db_config.url), Some(db_config.name)) + } else { + (None, None) + }; + + let (prover_db_url, prover_db_name) = if let Some(db_full_url) = database.prover_url { + let db_config = DatabaseConfig::from_url(db_full_url.expose_url()) + .context("Invalid prover database URL")?; + (Some(db_config.url), Some(db_config.name)) + } else { + (None, None) + }; + + self.server_db_url = self.server_db_url.or(server_db_url); + self.server_db_name = self.server_db_name.or(server_db_name); + self.prover_db_url = self.prover_db_url.or(prover_db_url); + self.prover_db_name = self.prover_db_name.or(prover_db_name); + + Ok(self.fill_values_with_prompt(chain_config)) + } + + pub fn reset_db_names(&mut self) { + self.prover_db_name = None; + self.prover_db_url = None; + self.server_db_name = None; + self.server_db_url = None; + } } #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init/configs.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init/configs.rs new file mode 100644 index 000000000000..b4a49f29d219 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init/configs.rs @@ -0,0 +1,70 @@ +use clap::Parser; +use common::Prompt; +use config::ChainConfig; +use serde::{Deserialize, Serialize}; +use types::L1Network; +use url::Url; + +use crate::{ + commands::chain::args::{ + genesis::{GenesisArgs, GenesisArgsFinal}, + init::InitArgsFinal, + }, + defaults::LOCAL_RPC_URL, + messages::{ + MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR, + MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP + }, +}; + +#[derive(Debug, Clone, Serialize, Deserialize, Parser)] +pub struct InitConfigsArgs { + #[clap(flatten, next_help_heading = MSG_GENESIS_ARGS_HELP)] + #[serde(flatten)] + pub genesis_args: GenesisArgs, + #[clap(long, help = MSG_L1_RPC_URL_HELP)] + pub l1_rpc_url: Option, + #[clap(long, help = MSG_NO_PORT_REALLOCATION_HELP, default_value = "false", default_missing_value = "true", num_args = 0..=1)] + pub no_port_reallocation: bool, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct InitConfigsArgsFinal { + pub genesis_args: GenesisArgsFinal, + pub l1_rpc_url: String, + pub no_port_reallocation: bool, +} + +impl InitConfigsArgs { + pub fn fill_values_with_prompt(self, config: &ChainConfig) -> InitConfigsArgsFinal { + let l1_rpc_url = self.l1_rpc_url.unwrap_or_else(|| { + let mut prompt = Prompt::new(MSG_L1_RPC_URL_PROMPT); + if config.l1_network == L1Network::Localhost { + prompt = prompt.default(LOCAL_RPC_URL); + } + prompt + .validate_with(|val: &String| -> Result<(), String> { + Url::parse(val) + .map(|_| ()) + .map_err(|_| MSG_L1_RPC_URL_INVALID_ERR.to_string()) + }) + .ask() + }); + + InitConfigsArgsFinal { + genesis_args: self.genesis_args.fill_values_with_prompt(config), + l1_rpc_url, + no_port_reallocation: self.no_port_reallocation, + } + } +} + +impl InitConfigsArgsFinal { + pub fn from_chain_init_args(init_args: &InitArgsFinal) -> InitConfigsArgsFinal { + InitConfigsArgsFinal { + genesis_args: init_args.genesis_args.clone(), + l1_rpc_url: init_args.l1_rpc_url.clone(), + no_port_reallocation: init_args.no_port_reallocation, + } + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init/mod.rs similarity index 96% rename from zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs rename to zk_toolbox/crates/zk_inception/src/commands/chain/args/init/mod.rs index 24a0539f27d1..be4d28202b80 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init/mod.rs @@ -5,9 +5,8 @@ use serde::{Deserialize, Serialize}; use types::L1Network; use url::Url; -use super::genesis::GenesisArgsFinal; use crate::{ - commands::chain::args::genesis::GenesisArgs, + commands::chain::args::genesis::{GenesisArgs, GenesisArgsFinal}, defaults::LOCAL_RPC_URL, messages::{ MSG_DEPLOY_PAYMASTER_PROMPT, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, @@ -15,6 +14,8 @@ use crate::{ }, }; +pub mod configs; + #[derive(Debug, Clone, Serialize, Deserialize, Parser)] pub struct InitArgs { /// All ethereum environment related arguments diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/build_transactions.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/build_transactions.rs index 98b2e226cc13..5f1be15231bf 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/build_transactions.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/build_transactions.rs @@ -6,9 +6,10 @@ use config::{ use ethers::utils::hex::ToHex; use xshell::Shell; -use super::common::register_chain; use crate::{ - commands::chain::args::build_transactions::BuildTransactionsArgs, + commands::chain::{ + args::build_transactions::BuildTransactionsArgs, register_chain::register_chain, + }, messages::{ MSG_BUILDING_CHAIN_REGISTRATION_TXNS_SPINNER, MSG_CHAIN_NOT_FOUND_ERR, MSG_CHAIN_TRANSACTIONS_BUILT, MSG_CHAIN_TXN_MISSING_CONTRACT_CONFIG, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/common.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/common.rs index 0aabc16154ee..e0aa0b4e0470 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/common.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/common.rs @@ -1,66 +1,12 @@ -use common::{ - forge::{Forge, ForgeScriptArgs}, - spinner::Spinner, -}; -use config::{ - forge_interface::{ - register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, - script_params::REGISTER_CHAIN_SCRIPT_PARAMS, - }, - traits::{ReadConfig, SaveConfig}, - ChainConfig, ContractsConfig, EcosystemConfig, -}; +use common::spinner::Spinner; +use config::{ChainConfig, EcosystemConfig}; use types::{BaseToken, L1Network, WalletCreation}; -use xshell::Shell; use crate::{ consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, messages::{MSG_DISTRIBUTING_ETH_SPINNER, MSG_MINT_BASE_TOKEN_SPINNER}, - utils::forge::{check_the_balance, fill_forge_private_key}, }; -#[allow(clippy::too_many_arguments)] -pub async fn register_chain( - shell: &Shell, - forge_args: ForgeScriptArgs, - config: &EcosystemConfig, - chain_config: &ChainConfig, - contracts: &mut ContractsConfig, - l1_rpc_url: String, - sender: Option, - broadcast: bool, -) -> anyhow::Result<()> { - let deploy_config_path = REGISTER_CHAIN_SCRIPT_PARAMS.input(&config.link_to_code); - - let deploy_config = RegisterChainL1Config::new(chain_config, contracts)?; - deploy_config.save(shell, deploy_config_path)?; - - let mut forge = Forge::new(&config.path_to_foundry()) - .script(®ISTER_CHAIN_SCRIPT_PARAMS.script(), forge_args.clone()) - .with_ffi() - .with_rpc_url(l1_rpc_url); - - if broadcast { - forge = forge.with_broadcast(); - } - - if let Some(address) = sender { - forge = forge.with_sender(address); - } else { - forge = fill_forge_private_key(forge, config.get_wallets()?.governor_private_key())?; - check_the_balance(&forge).await?; - } - - forge.run(shell)?; - - let register_chain_output = RegisterChainOutput::read( - shell, - REGISTER_CHAIN_SCRIPT_PARAMS.output(&chain_config.link_to_code), - )?; - contracts.set_chain_contracts(®ister_chain_output); - Ok(()) -} - // Distribute eth to the chain wallets for localhost environment pub async fn distribute_eth( ecosystem_config: &EcosystemConfig, diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs similarity index 63% rename from zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs rename to zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs index c72183e98b77..bb78979ec38e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs @@ -5,33 +5,26 @@ use common::{ config::global_config, db::{drop_db_if_exists, init_db, migrate_db, DatabaseConfig}, logger, - server::{Server, ServerMode}, - spinner::Spinner, }; use config::{ override_config, set_databases, set_file_artifacts, set_rocks_db_config, - traits::{FileConfigWithDefaultName, SaveConfigWithBasePath}, - ChainConfig, ContractsConfig, EcosystemConfig, FileArtifacts, GeneralConfig, GenesisConfig, - SecretsConfig, WalletsConfig, + traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig, FileArtifacts, }; use types::ProverMode; use xshell::Shell; use zksync_basic_types::commitment::L1BatchCommitmentMode; -use super::args::genesis::GenesisArgsFinal; use crate::{ - commands::chain::args::genesis::GenesisArgs, + commands::chain::args::genesis::{GenesisArgs, GenesisArgsFinal}, consts::{ PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG, PATH_TO_VALIDIUM_OVERRIDE_CONFIG, PROVER_MIGRATIONS, SERVER_MIGRATIONS, }, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, - MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_FAILED_TO_RUN_SERVER_ERR, - MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, + MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_DATABASES_INITIALIZED, MSG_INITIALIZING_PROVER_DATABASE, MSG_INITIALIZING_SERVER_DATABASE, - MSG_RECREATE_ROCKS_DB_ERRROR, MSG_SELECTED_CONFIG, MSG_STARTING_GENESIS, - MSG_STARTING_GENESIS_SPINNER, + MSG_RECREATE_ROCKS_DB_ERRROR, }, utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }; @@ -41,79 +34,26 @@ pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let chain_config = ecosystem_config .load_current_chain() .context(MSG_CHAIN_NOT_INITIALIZED)?; - let args = args.fill_values_with_prompt(&chain_config); - genesis(args, shell, &chain_config).await?; - logger::outro(MSG_GENESIS_COMPLETED); - - Ok(()) -} - -pub async fn genesis( - args: GenesisArgsFinal, - shell: &Shell, - config: &ChainConfig, -) -> anyhow::Result<()> { - shell.create_dir(&config.rocks_db_path)?; - - let link_to_code = config.link_to_code.clone(); - let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) - .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; - let mut general = config.get_general_config()?; - let file_artifacts = FileArtifacts::new(config.artifacts.clone()); - set_rocks_db_config(&mut general, rocks_db)?; - set_file_artifacts(&mut general, file_artifacts); - general.save_with_base_path(shell, &config.configs)?; - - if config.prover_version != ProverMode::NoProofs { - override_config( - shell, - link_to_code.join(PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG), - config, - )?; - } - - if config.l1_batch_commit_data_generator_mode == L1BatchCommitmentMode::Validium { - override_config( - shell, - link_to_code.join(PATH_TO_VALIDIUM_OVERRIDE_CONFIG), - config, - )?; - } - - let mut secrets = config.get_secrets_config()?; + let mut secrets = chain_config.get_secrets_config()?; + let args = args.fill_values_with_secrets(&chain_config)?; set_databases(&mut secrets, &args.server_db, &args.prover_db)?; - secrets.save_with_base_path(shell, &config.configs)?; - - logger::note( - MSG_SELECTED_CONFIG, - logger::object_to_string(serde_json::json!({ - "chain_config": config, - "server_db_config": args.server_db, - "prover_db_config": args.prover_db, - })), - ); - logger::info(MSG_STARTING_GENESIS); + secrets.save_with_base_path(shell, &chain_config.configs)?; - let spinner = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); initialize_databases( shell, &args.server_db, &args.prover_db, - config.link_to_code.clone(), + chain_config.link_to_code.clone(), args.dont_drop, ) .await?; - spinner.finish(); - - let spinner = Spinner::new(MSG_STARTING_GENESIS_SPINNER); - run_server_genesis(config, shell)?; - spinner.finish(); + logger::outro(MSG_GENESIS_DATABASES_INITIALIZED); Ok(()) } -async fn initialize_databases( +pub async fn initialize_databases( shell: &Shell, server_db_config: &DatabaseConfig, prover_db_config: &DatabaseConfig, @@ -158,18 +98,41 @@ async fn initialize_databases( Ok(()) } -fn run_server_genesis(chain_config: &ChainConfig, shell: &Shell) -> anyhow::Result<()> { - let server = Server::new(None, chain_config.link_to_code.clone(), false); - server - .run( +pub fn update_configs( + args: GenesisArgsFinal, + shell: &Shell, + config: &ChainConfig, +) -> anyhow::Result<()> { + shell.create_dir(&config.rocks_db_path)?; + + // Update secrets configs + let mut secrets = config.get_secrets_config()?; + set_databases(&mut secrets, &args.server_db, &args.prover_db)?; + secrets.save_with_base_path(shell, &config.configs)?; + + // Update general config + let mut general = config.get_general_config()?; + let rocks_db = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::Main) + .context(MSG_RECREATE_ROCKS_DB_ERRROR)?; + let file_artifacts = FileArtifacts::new(config.artifacts.clone()); + set_rocks_db_config(&mut general, rocks_db)?; + set_file_artifacts(&mut general, file_artifacts); + general.save_with_base_path(shell, &config.configs)?; + + let link_to_code = config.link_to_code.clone(); + if config.prover_version != ProverMode::NoProofs { + override_config( + shell, + link_to_code.join(PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG), + config, + )?; + } + if config.l1_batch_commit_data_generator_mode == L1BatchCommitmentMode::Validium { + override_config( shell, - ServerMode::Genesis, - GenesisConfig::get_path_with_base_path(&chain_config.configs), - WalletsConfig::get_path_with_base_path(&chain_config.configs), - GeneralConfig::get_path_with_base_path(&chain_config.configs), - SecretsConfig::get_path_with_base_path(&chain_config.configs), - ContractsConfig::get_path_with_base_path(&chain_config.configs), - vec![], - ) - .context(MSG_FAILED_TO_RUN_SERVER_ERR) + link_to_code.join(PATH_TO_VALIDIUM_OVERRIDE_CONFIG), + config, + )?; + } + Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs new file mode 100644 index 000000000000..01842c2916ae --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs @@ -0,0 +1,94 @@ +use anyhow::Context; +use clap::{command, Parser, Subcommand}; +use common::{logger, spinner::Spinner}; +use config::{ChainConfig, EcosystemConfig}; +use xshell::Shell; + +use crate::{ + commands::chain::{ + args::genesis::{GenesisArgs, GenesisArgsFinal}, + genesis::{self, database::initialize_databases, server::run_server_genesis}, + }, + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, + MSG_SELECTED_CONFIG, MSG_STARTING_GENESIS, MSG_STARTING_GENESIS_SPINNER, + }, +}; + +// Genesis subcommands +pub mod database; +pub mod server; + +#[derive(Subcommand, Debug, Clone)] +pub enum GenesisSubcommands { + /// Initialize databases + #[command(alias = "database")] + InitDatabase(Box), + /// Runs server genesis + Server, +} + +#[derive(Parser, Debug)] +#[command()] +pub struct GenesisCommand { + #[command(subcommand)] + command: Option, + #[clap(flatten)] + args: GenesisArgs, +} + +pub(crate) async fn run(args: GenesisCommand, shell: &Shell) -> anyhow::Result<()> { + match args.command { + Some(GenesisSubcommands::InitDatabase(args)) => database::run(*args, shell).await, + Some(GenesisSubcommands::Server) => server::run(shell).await, + None => run_genesis(args.args, shell).await, + } +} + +pub async fn run_genesis(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_current_chain() + .context(MSG_CHAIN_NOT_INITIALIZED)?; + let args = args.fill_values_with_prompt(&chain_config); + + genesis(args, shell, &chain_config).await?; + logger::outro(MSG_GENESIS_COMPLETED); + + Ok(()) +} + +pub async fn genesis( + args: GenesisArgsFinal, + shell: &Shell, + config: &ChainConfig, +) -> anyhow::Result<()> { + genesis::database::update_configs(args.clone(), shell, config)?; + + logger::note( + MSG_SELECTED_CONFIG, + logger::object_to_string(serde_json::json!({ + "chain_config": config, + "server_db_config": args.server_db, + "prover_db_config": args.prover_db, + })), + ); + logger::info(MSG_STARTING_GENESIS); + + let spinner = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); + initialize_databases( + shell, + &args.server_db, + &args.prover_db, + config.link_to_code.clone(), + args.dont_drop, + ) + .await?; + spinner.finish(); + + let spinner = Spinner::new(MSG_STARTING_GENESIS_SPINNER); + run_server_genesis(config, shell)?; + spinner.finish(); + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/server.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/server.rs new file mode 100644 index 000000000000..50a74b7ea9e4 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/server.rs @@ -0,0 +1,46 @@ +use anyhow::Context; +use common::{ + logger, + server::{Server, ServerMode}, + spinner::Spinner, +}; +use config::{ + traits::FileConfigWithDefaultName, ChainConfig, ContractsConfig, EcosystemConfig, + GeneralConfig, GenesisConfig, SecretsConfig, WalletsConfig, +}; +use xshell::Shell; + +use crate::messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_RUN_SERVER_ERR, MSG_GENESIS_COMPLETED, + MSG_STARTING_GENESIS_SPINNER, +}; + +pub async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_current_chain() + .context(MSG_CHAIN_NOT_INITIALIZED)?; + + let spinner = Spinner::new(MSG_STARTING_GENESIS_SPINNER); + run_server_genesis(&chain_config, shell)?; + spinner.finish(); + logger::outro(MSG_GENESIS_COMPLETED); + + Ok(()) +} + +pub fn run_server_genesis(chain_config: &ChainConfig, shell: &Shell) -> anyhow::Result<()> { + let server = Server::new(None, chain_config.link_to_code.clone(), false); + server + .run( + shell, + ServerMode::Genesis, + GenesisConfig::get_path_with_base_path(&chain_config.configs), + WalletsConfig::get_path_with_base_path(&chain_config.configs), + GeneralConfig::get_path_with_base_path(&chain_config.configs), + SecretsConfig::get_path_with_base_path(&chain_config.configs), + ContractsConfig::get_path_with_base_path(&chain_config.configs), + vec![], + ) + .context(MSG_FAILED_TO_RUN_SERVER_ERR) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs new file mode 100644 index 000000000000..e6b9fa7117d4 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs @@ -0,0 +1,107 @@ +use anyhow::Context; +use common::logger; +use config::{ + copy_configs, set_l1_rpc_url, update_from_chain_config, + ChainConfig, ContractsConfig, EcosystemConfig, + traits::SaveConfigWithBasePath, + DEFAULT_CONSENSUS_PORT, +}; +use ethers::types::Address; +use xshell::Shell; + +use crate::{ + commands::{ + chain::{ + args::init::configs::{InitConfigsArgs, InitConfigsArgsFinal}, + genesis, + }, + portal::update_portal_config, + }, + defaults::PORT_RANGE_END, + messages::{ + MSG_CHAIN_CONFIGS_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, + MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, + }, + utils::{ + consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets}, + ports::EcosystemPortsScanner, + }, +}; + +pub async fn run(args: InitConfigsArgs, shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_current_chain() + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + let args = args.fill_values_with_prompt(&chain_config); + + init_configs(&args, shell, &ecosystem_config, &chain_config).await?; + logger::outro(MSG_CHAIN_CONFIGS_INITIALIZED); + + Ok(()) +} + +pub async fn init_configs( + init_args: &InitConfigsArgsFinal, + shell: &Shell, + ecosystem_config: &EcosystemConfig, + chain_config: &ChainConfig, +) -> anyhow::Result { + // Port scanner should run before copying configs to avoid marking initial ports as assigned + let mut ecosystem_ports = EcosystemPortsScanner::scan(shell)?; + copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; + + if !init_args.no_port_reallocation { + ecosystem_ports.allocate_ports_in_yaml( + shell, + &chain_config.path_to_general_config(), + chain_config.id, + )?; + } + + // Initialize general config + let mut general_config = chain_config.get_general_config()?; + + // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` + let offset = ((chain_config.id - 1) * 100) as u16; + let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; + let consensus_port = + ecosystem_ports.allocate_port(consensus_port_range, "Consensus".to_string())?; + + let consensus_keys = generate_consensus_keys(); + let consensus_config = get_consensus_config( + chain_config, + consensus_port, + Some(consensus_keys.clone()), + None, + )?; + general_config.consensus_config = Some(consensus_config); + general_config.save_with_base_path(shell, &chain_config.configs)?; + + // Initialize genesis config + let mut genesis_config = chain_config.get_genesis_config()?; + update_from_chain_config(&mut genesis_config, chain_config); + genesis_config.save_with_base_path(shell, &chain_config.configs)?; + + // Initialize contracts config + let mut contracts_config = ecosystem_config.get_contracts_config()?; + contracts_config.l1.diamond_proxy_addr = Address::zero(); + contracts_config.l1.governance_addr = Address::zero(); + contracts_config.l1.chain_admin_addr = Address::zero(); + contracts_config.l1.base_token_addr = chain_config.base_token.address; + contracts_config.save_with_base_path(shell, &chain_config.configs)?; + + // Initialize secrets config + let mut secrets = chain_config.get_secrets_config()?; + set_l1_rpc_url(&mut secrets, init_args.l1_rpc_url.clone())?; + secrets.consensus = Some(get_consensus_secrets(&consensus_keys)); + secrets.save_with_base_path(shell, &chain_config.configs)?; + + genesis::database::update_configs(init_args.genesis_args.clone(), shell, chain_config)?; + + update_portal_config(shell, chain_config) + .await + .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; + + Ok(contracts_config) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init/mod.rs similarity index 55% rename from zk_toolbox/crates/zk_inception/src/commands/chain/init.rs rename to zk_toolbox/crates/zk_inception/src/commands/chain/init/mod.rs index 4d7096723da2..8a36f4e32b2f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init/mod.rs @@ -1,106 +1,91 @@ use anyhow::Context; +use clap::{command, Parser, Subcommand}; use common::{git, logger, spinner::Spinner}; -use config::{ - copy_configs, set_l1_rpc_url, traits::SaveConfigWithBasePath, update_from_chain_config, - ChainConfig, EcosystemConfig, DEFAULT_CONSENSUS_PORT, -}; +use config::{traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig}; use types::BaseToken; use xshell::Shell; -use super::common::{distribute_eth, mint_base_token, register_chain}; use crate::{ accept_ownership::accept_admin, - commands::{ - chain::{ - args::init::{InitArgs, InitArgsFinal}, - deploy_l2_contracts, deploy_paymaster, - genesis::genesis, - set_token_multiplier_setter::set_token_multiplier_setter, - setup_legacy_bridge::setup_legacy_bridge, + commands::chain::{ + args::init::{ + configs::{InitConfigsArgs, InitConfigsArgsFinal}, + InitArgs, InitArgsFinal, }, - portal::update_portal_config, + common::{distribute_eth, mint_base_token}, + deploy_l2_contracts, deploy_paymaster, + genesis::genesis, + init::configs::init_configs, + register_chain::register_chain, + set_token_multiplier_setter::set_token_multiplier_setter, + setup_legacy_bridge::setup_legacy_bridge, }, - defaults::PORT_RANGE_END, messages::{ msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_DEPLOYING_PAYMASTER, MSG_GENESIS_DATABASE_ERR, - MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, + MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG, MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND, }, - utils::{ - consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets}, - ports::EcosystemPortsScanner, - }, }; -pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { +// Init subcommands +pub mod configs; + +#[derive(Subcommand, Debug, Clone)] +pub enum ChainInitSubcommands { + /// Initialize chain configs + Configs(InitConfigsArgs), +} + +#[derive(Parser, Debug)] +#[command()] +pub struct ChainInitCommand { + #[command(subcommand)] + command: Option, + #[clap(flatten)] + args: InitArgs, +} + +pub(crate) async fn run(args: ChainInitCommand, shell: &Shell) -> anyhow::Result<()> { + match args.command { + Some(ChainInitSubcommands::Configs(args)) => configs::run(args, shell).await, + None => run_init(args.args, shell).await, + } +} + +async fn run_init(args: InitArgs, shell: &Shell) -> anyhow::Result<()> { let config = EcosystemConfig::from_file(shell)?; let chain_config = config .load_current_chain() .context(MSG_CHAIN_NOT_FOUND_ERR)?; - let mut args = args.fill_values_with_prompt(&chain_config); + let args = args.fill_values_with_prompt(&chain_config); logger::note(MSG_SELECTED_CONFIG, logger::object_to_string(&chain_config)); logger::info(msg_initializing_chain("")); git::submodule_update(shell, config.link_to_code.clone())?; - init(&mut args, shell, &config, &chain_config).await?; + init(&args, shell, &config, &chain_config).await?; logger::success(MSG_CHAIN_INITIALIZED); Ok(()) } pub async fn init( - init_args: &mut InitArgsFinal, + init_args: &InitArgsFinal, shell: &Shell, ecosystem_config: &EcosystemConfig, chain_config: &ChainConfig, ) -> anyhow::Result<()> { - let mut ecosystem_ports = EcosystemPortsScanner::scan(shell)?; - copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?; - - if !init_args.no_port_reallocation { - ecosystem_ports.allocate_ports_in_yaml( - shell, - &chain_config.path_to_general_config(), - chain_config.id, - )?; - } - let mut general_config = chain_config.get_general_config()?; - - // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - let offset = ((chain_config.id - 1) * 100) as u16; - let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; - let consensus_port = - ecosystem_ports.allocate_port(consensus_port_range, "Consensus".to_string())?; - - let consensus_keys = generate_consensus_keys(); - let consensus_config = get_consensus_config( - chain_config, - consensus_port, - Some(consensus_keys.clone()), - None, - )?; - general_config.consensus_config = Some(consensus_config); - general_config.save_with_base_path(shell, &chain_config.configs)?; - - let mut genesis_config = chain_config.get_genesis_config()?; - update_from_chain_config(&mut genesis_config, chain_config); - genesis_config.save_with_base_path(shell, &chain_config.configs)?; - - // Copy ecosystem contracts - let mut contracts_config = ecosystem_config.get_contracts_config()?; - contracts_config.l1.base_token_addr = chain_config.base_token.address; - contracts_config.save_with_base_path(shell, &chain_config.configs)?; - + // Initialize configs + let init_configs_args = InitConfigsArgsFinal::from_chain_init_args(init_args); + let mut contracts_config = + init_configs(&init_configs_args, shell, ecosystem_config, chain_config).await?; + + // Fund some wallet addresses with ETH or base token (only for Localhost) distribute_eth(ecosystem_config, chain_config, init_args.l1_rpc_url.clone()).await?; mint_base_token(ecosystem_config, chain_config, init_args.l1_rpc_url.clone()).await?; - let mut secrets = chain_config.get_secrets_config()?; - set_l1_rpc_url(&mut secrets, init_args.l1_rpc_url.clone())?; - secrets.consensus = Some(get_consensus_secrets(&consensus_keys)); - secrets.save_with_base_path(shell, &chain_config.configs)?; - + // Register chain on BridgeHub (run by L1 Governor) let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); register_chain( shell, @@ -115,6 +100,8 @@ pub async fn init( .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; spinner.finish(); + + // Accept ownership for DiamondProxy (run by L2 Governor) let spinner = Spinner::new(MSG_ACCEPTING_ADMIN_SPINNER); accept_admin( shell, @@ -128,6 +115,7 @@ pub async fn init( .await?; spinner.finish(); + // Set token multiplier setter address (run by L2 Governor) if chain_config.base_token != BaseToken::eth() { let spinner = Spinner::new(MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER); set_token_multiplier_setter( @@ -148,6 +136,7 @@ pub async fn init( spinner.finish(); } + // Deploy L2 contracts: L2SharedBridge, L2DefaultUpgrader, ... (run by L1 Governor) deploy_l2_contracts::deploy_l2_contracts( shell, chain_config, @@ -158,6 +147,7 @@ pub async fn init( .await?; contracts_config.save_with_base_path(shell, &chain_config.configs)?; + // Setup legacy bridge - shouldn't be used for new chains (run by L1 Governor) if let Some(true) = chain_config.legacy_bridge { setup_legacy_bridge( shell, @@ -169,6 +159,7 @@ pub async fn init( .await?; } + // Deploy Paymaster contract (run by L2 Governor) if init_args.deploy_paymaster { let spinner = Spinner::new(MSG_DEPLOYING_PAYMASTER); deploy_paymaster::deploy_paymaster( @@ -187,10 +178,6 @@ pub async fn init( genesis(init_args.genesis_args.clone(), shell, chain_config) .await .context(MSG_GENESIS_DATABASE_ERR)?; - - update_portal_config(shell, chain_config) - .await - .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; - + Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs index 4ddc4bf58569..378309a07cb1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/mod.rs @@ -1,15 +1,16 @@ use ::common::forge::ForgeScriptArgs; use args::build_transactions::BuildTransactionsArgs; pub(crate) use args::create::ChainCreateArgsFinal; -use clap::Subcommand; +use clap::{command, Subcommand}; pub(crate) use create::create_chain_inner; use xshell::Shell; use crate::commands::chain::{ - args::{create::ChainCreateArgs, genesis::GenesisArgs, init::InitArgs}, - deploy_l2_contracts::Deploy2ContractsOption, + args::create::ChainCreateArgs, deploy_l2_contracts::Deploy2ContractsOption, + genesis::GenesisCommand, init::ChainInitCommand, }; +mod accept_chain_ownership; pub(crate) mod args; mod build_transactions; mod common; @@ -17,7 +18,8 @@ mod create; pub mod deploy_l2_contracts; pub mod deploy_paymaster; pub mod genesis; -pub(crate) mod init; +pub mod init; +pub mod register_chain; mod set_token_multiplier_setter; mod setup_legacy_bridge; @@ -28,20 +30,32 @@ pub enum ChainCommands { /// Create unsigned transactions for chain deployment BuildTransactions(BuildTransactionsArgs), /// Initialize chain, deploying necessary contracts and performing on-chain operations - Init(InitArgs), + Init(Box), /// Run server genesis - Genesis(GenesisArgs), - /// Initialize bridges on l2 - #[command(alias = "bridge")] - InitializeBridges(ForgeScriptArgs), - /// Deploy all l2 contracts + Genesis(GenesisCommand), + /// Register a new chain on L1 (executed by L1 governor). + /// This command deploys and configures Governance, ChainAdmin, and DiamondProxy contracts, + /// registers chain with BridgeHub and sets pending admin for DiamondProxy. + /// Note: After completion, L2 governor can accept ownership by running `accept-chain-ownership` + #[command(alias = "register")] + RegisterChain(ForgeScriptArgs), + /// Deploy all L2 contracts (executed by L1 governor). #[command(alias = "l2")] DeployL2Contracts(ForgeScriptArgs), + /// Accept ownership of L2 chain (executed by L2 governor). + /// This command should be run after `register-chain` to accept ownership of newly created + /// DiamondProxy contract. + #[command(alias = "accept-ownership")] + AcceptChainOwnership(ForgeScriptArgs), + /// Initialize bridges on L2 + #[command(alias = "bridge")] + InitializeBridges(ForgeScriptArgs), /// Deploy L2 consensus registry #[command(alias = "consensus")] DeployConsensusRegistry(ForgeScriptArgs), /// Deploy Default Upgrader - Upgrader(ForgeScriptArgs), + #[command(alias = "upgrader")] + DeployUpgrader(ForgeScriptArgs), /// Deploy paymaster smart contract #[command(alias = "paymaster")] DeployPaymaster(ForgeScriptArgs), @@ -52,16 +66,18 @@ pub enum ChainCommands { pub(crate) async fn run(shell: &Shell, args: ChainCommands) -> anyhow::Result<()> { match args { ChainCommands::Create(args) => create::run(args, shell), - ChainCommands::Init(args) => init::run(args, shell).await, + ChainCommands::Init(args) => init::run(*args, shell).await, ChainCommands::BuildTransactions(args) => build_transactions::run(args, shell).await, ChainCommands::Genesis(args) => genesis::run(args, shell).await, + ChainCommands::RegisterChain(args) => register_chain::run(args, shell).await, ChainCommands::DeployL2Contracts(args) => { deploy_l2_contracts::run(args, shell, Deploy2ContractsOption::All).await } + ChainCommands::AcceptChainOwnership(args) => accept_chain_ownership::run(args, shell).await, ChainCommands::DeployConsensusRegistry(args) => { deploy_l2_contracts::run(args, shell, Deploy2ContractsOption::ConsensusRegistry).await } - ChainCommands::Upgrader(args) => { + ChainCommands::DeployUpgrader(args) => { deploy_l2_contracts::run(args, shell, Deploy2ContractsOption::Upgrader).await } ChainCommands::InitializeBridges(args) => { diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/register_chain.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/register_chain.rs new file mode 100644 index 000000000000..9f2ff41f897e --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/register_chain.rs @@ -0,0 +1,96 @@ +use anyhow::Context; +use common::{ + forge::{Forge, ForgeScriptArgs}, + logger, + spinner::Spinner, +}; +use config::{ + forge_interface::{ + register_chain::{input::RegisterChainL1Config, output::RegisterChainOutput}, + script_params::REGISTER_CHAIN_SCRIPT_PARAMS, + }, + traits::{ReadConfig, SaveConfig, SaveConfigWithBasePath}, + ChainConfig, ContractsConfig, EcosystemConfig, +}; +use xshell::Shell; + +use crate::{ + messages::{ + MSG_CHAIN_NOT_INITIALIZED, MSG_CHAIN_REGISTERED, MSG_L1_SECRETS_MUST_BE_PRESENTED, + MSG_REGISTERING_CHAIN_SPINNER, + }, + utils::forge::{check_the_balance, fill_forge_private_key}, +}; + +pub async fn run(args: ForgeScriptArgs, shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_current_chain() + .context(MSG_CHAIN_NOT_INITIALIZED)?; + let mut contracts = chain_config.get_contracts_config()?; + let secrets = chain_config.get_secrets_config()?; + let l1_rpc_url = secrets + .l1 + .context(MSG_L1_SECRETS_MUST_BE_PRESENTED)? + .l1_rpc_url + .expose_str() + .to_string(); + let spinner = Spinner::new(MSG_REGISTERING_CHAIN_SPINNER); + register_chain( + shell, + args, + &ecosystem_config, + &chain_config, + &mut contracts, + l1_rpc_url, + None, + true, + ) + .await?; + contracts.save_with_base_path(shell, chain_config.configs)?; + spinner.finish(); + logger::success(MSG_CHAIN_REGISTERED); + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub async fn register_chain( + shell: &Shell, + forge_args: ForgeScriptArgs, + config: &EcosystemConfig, + chain_config: &ChainConfig, + contracts: &mut ContractsConfig, + l1_rpc_url: String, + sender: Option, + broadcast: bool, +) -> anyhow::Result<()> { + let deploy_config_path = REGISTER_CHAIN_SCRIPT_PARAMS.input(&config.link_to_code); + + let deploy_config = RegisterChainL1Config::new(chain_config, contracts)?; + deploy_config.save(shell, deploy_config_path)?; + + let mut forge = Forge::new(&config.path_to_foundry()) + .script(®ISTER_CHAIN_SCRIPT_PARAMS.script(), forge_args.clone()) + .with_ffi() + .with_rpc_url(l1_rpc_url); + + if broadcast { + forge = forge.with_broadcast(); + } + + if let Some(address) = sender { + forge = forge.with_sender(address); + } else { + forge = fill_forge_private_key(forge, config.get_wallets()?.governor_private_key())?; + check_the_balance(&forge).await?; + } + + forge.run(shell)?; + + let register_chain_output = RegisterChainOutput::read( + shell, + REGISTER_CHAIN_SCRIPT_PARAMS.output(&chain_config.link_to_code), + )?; + contracts.set_chain_contracts(®ister_chain_output); + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs index 7898f8d254a8..6d6ed2f3fd9e 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs @@ -10,10 +10,10 @@ use crate::{ commands::chain::args::genesis::GenesisArgs, defaults::LOCAL_RPC_URL, messages::{ - MSG_DEPLOY_ECOSYSTEM_PROMPT, MSG_DEPLOY_ERC20_PROMPT, MSG_DEPLOY_PAYMASTER_PROMPT, - MSG_DEV_ARG_HELP, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR, - MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP, MSG_OBSERVABILITY_HELP, - MSG_OBSERVABILITY_PROMPT, + MSG_DEPLOY_ECOSYSTEM_PROMPT, MSG_DEPLOY_ERC20_PROMPT, MSG_DEV_ARG_HELP, + MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR, + MSG_L1_RPC_URL_PROMPT, MSG_OBSERVABILITY_HELP, MSG_OBSERVABILITY_PROMPT, + MSG_NO_PORT_REALLOCATION_HELP, }, }; @@ -74,9 +74,6 @@ pub struct EcosystemArgsFinal { #[derive(Debug, Clone, Serialize, Deserialize, Parser)] pub struct EcosystemInitArgs { - /// Deploy Paymaster contract - #[clap(long, default_missing_value = "true", num_args = 0..=1)] - pub deploy_paymaster: Option, /// Deploy ERC20 contracts #[clap(long, default_missing_value = "true", num_args = 0..=1)] pub deploy_erc20: Option, @@ -86,9 +83,15 @@ pub struct EcosystemInitArgs { #[clap(flatten)] #[serde(flatten)] pub forge_args: ForgeScriptArgs, + /// Deploy Paymaster contract + #[clap(long, default_missing_value = "true", num_args = 0..=1)] + pub deploy_paymaster: Option, #[clap(flatten, next_help_heading = MSG_GENESIS_ARGS_HELP)] #[serde(flatten)] pub genesis_args: GenesisArgs, + /// Initialize ecosystem only and skip chain initialization (chain can be initialized later with `chain init` subcommand) + #[clap(long, default_value_t = false)] + pub ecosystem_only: bool, #[clap(long, help = MSG_DEV_ARG_HELP)] pub dev: bool, #[clap(long, short = 'o', help = MSG_OBSERVABILITY_HELP, default_missing_value = "true", num_args = 0..=1)] @@ -99,20 +102,14 @@ pub struct EcosystemInitArgs { impl EcosystemInitArgs { pub fn fill_values_with_prompt(self, l1_network: L1Network) -> EcosystemInitArgsFinal { - let (deploy_paymaster, deploy_erc20) = if self.dev { - (true, true) + let deploy_erc20 = if self.dev { + true } else { - let deploy_paymaster = self.deploy_paymaster.unwrap_or_else(|| { - PromptConfirm::new(MSG_DEPLOY_PAYMASTER_PROMPT) - .default(true) - .ask() - }); - let deploy_erc20 = self.deploy_erc20.unwrap_or_else(|| { + self.deploy_erc20.unwrap_or_else(|| { PromptConfirm::new(MSG_DEPLOY_ERC20_PROMPT) .default(true) .ask() - }); - (deploy_paymaster, deploy_erc20) + }) }; let ecosystem = self.ecosystem.fill_values_with_prompt(l1_network, self.dev); let observability = if self.dev { @@ -126,12 +123,12 @@ impl EcosystemInitArgs { }; EcosystemInitArgsFinal { - deploy_paymaster, deploy_erc20, ecosystem, forge_args: self.forge_args.clone(), dev: self.dev, observability, + ecosystem_only: self.ecosystem_only, no_port_reallocation: self.no_port_reallocation, } } @@ -139,11 +136,11 @@ impl EcosystemInitArgs { #[derive(Debug, Serialize, Deserialize)] pub struct EcosystemInitArgsFinal { - pub deploy_paymaster: bool, pub deploy_erc20: bool, pub ecosystem: EcosystemArgsFinal, pub forge_args: ForgeScriptArgs, pub dev: bool, pub observability: bool, + pub ecosystem_only: bool, pub no_port_reallocation: bool, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs index 80efc48f732b..67ae31628423 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs @@ -37,9 +37,8 @@ use crate::{ }, }, messages::{ - msg_ecosystem_initialized, msg_ecosystem_no_found_preexisting_contract, - msg_initializing_chain, MSG_CHAIN_NOT_INITIALIZED, - MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, + msg_chain_load_err, msg_ecosystem_initialized, msg_ecosystem_no_found_preexisting_contract, + msg_initializing_chain, MSG_DEPLOYING_ECOSYSTEM_CONTRACTS_SPINNER, MSG_DEPLOYING_ERC20, MSG_DEPLOYING_ERC20_SPINNER, MSG_ECOSYSTEM_CONTRACTS_PATH_INVALID_ERR, MSG_ECOSYSTEM_CONTRACTS_PATH_PROMPT, MSG_INITIALIZING_ECOSYSTEM, MSG_INTALLING_DEPS_SPINNER, @@ -57,11 +56,9 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { Err(_) => create_initial_deployments_config(shell, &ecosystem_config.config)?, }; - let mut genesis_args = args.genesis_args.clone(); - if args.dev { - genesis_args.use_default = true; - } - let mut final_ecosystem_args = args.fill_values_with_prompt(ecosystem_config.l1_network); + let mut final_ecosystem_args = args + .clone() + .fill_values_with_prompt(ecosystem_config.l1_network); logger::info(MSG_INITIALIZING_ECOSYSTEM); @@ -69,7 +66,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { setup_observability::run(shell)?; } - let contracts_config = init( + let contracts_config = init_ecosystem( &mut final_ecosystem_args, shell, &ecosystem_config, @@ -94,42 +91,17 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> { .await?; } - // If the name of chain passed then we deploy exactly this chain otherwise deploy all chains - let list_of_chains = if let Some(name) = global_config().chain_name.clone() { - vec![name] - } else { - ecosystem_config.list_of_chains() - }; - - for chain_name in &list_of_chains { - logger::info(msg_initializing_chain(chain_name)); - let chain_config = ecosystem_config - .load_chain(Some(chain_name.clone())) - .context(MSG_CHAIN_NOT_INITIALIZED)?; - - let mut chain_init_args = chain::args::init::InitArgsFinal { - forge_args: final_ecosystem_args.forge_args.clone(), - genesis_args: genesis_args.clone().fill_values_with_prompt(&chain_config), - deploy_paymaster: final_ecosystem_args.deploy_paymaster, - l1_rpc_url: final_ecosystem_args.ecosystem.l1_rpc_url.clone(), - no_port_reallocation: final_ecosystem_args.no_port_reallocation, - }; - - chain::init::init( - &mut chain_init_args, - shell, - &ecosystem_config, - &chain_config, - ) - .await?; + // Initialize chain(s) + let mut chains: Vec = vec![]; + if !final_ecosystem_args.ecosystem_only { + chains = init_chains(&args, &final_ecosystem_args, shell, &ecosystem_config).await?; } - - logger::outro(msg_ecosystem_initialized(&list_of_chains.join(","))); + logger::outro(msg_ecosystem_initialized(&chains.join(","))); Ok(()) } -async fn init( +async fn init_ecosystem( init_args: &mut EcosystemInitArgsFinal, shell: &Shell, ecosystem_config: &EcosystemConfig, @@ -358,3 +330,53 @@ async fn deploy_ecosystem_inner( Ok(contracts_config) } + +async fn init_chains( + init_args: &EcosystemInitArgs, + final_init_args: &EcosystemInitArgsFinal, + shell: &Shell, + ecosystem_config: &EcosystemConfig, +) -> anyhow::Result> { + // If the name of chain passed then we deploy exactly this chain otherwise deploy all chains + let list_of_chains = if let Some(name) = global_config().chain_name.clone() { + vec![name] + } else { + ecosystem_config.list_of_chains() + }; + // Set default values for dev mode + let mut deploy_paymaster = init_args.deploy_paymaster; + let mut genesis_args = init_args.genesis_args.clone(); + if final_init_args.dev { + deploy_paymaster = Some(true); + genesis_args.use_default = true; + } + // Can't initialize multiple chains with the same DB + if list_of_chains.len() > 1 { + genesis_args.reset_db_names(); + } + // Initialize chains + for chain_name in &list_of_chains { + logger::info(msg_initializing_chain(chain_name)); + let chain_config = ecosystem_config + .load_chain(Some(chain_name.clone())) + .context(msg_chain_load_err(chain_name))?; + + let chain_init_args = chain::args::init::InitArgs { + forge_args: final_init_args.forge_args.clone(), + genesis_args: genesis_args.clone(), + deploy_paymaster, + l1_rpc_url: Some(final_init_args.ecosystem.l1_rpc_url.clone()), + no_port_reallocation: final_init_args.no_port_reallocation + }; + let final_chain_init_args = chain_init_args.fill_values_with_prompt(&chain_config); + + chain::init::init( + &final_chain_init_args, + shell, + ecosystem_config, + &chain_config, + ) + .await?; + } + Ok(list_of_chains) +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index 0af9922d0c41..a305ca053b7c 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -42,10 +42,10 @@ struct Inception { pub enum InceptionSubcommands { /// Ecosystem related commands #[command(subcommand, alias = "e")] - Ecosystem(EcosystemCommands), + Ecosystem(Box), /// Chain related commands #[command(subcommand, alias = "c")] - Chain(ChainCommands), + Chain(Box), /// Prover related commands #[command(subcommand, alias = "p")] Prover(ProverCommands), @@ -121,8 +121,8 @@ async fn main() -> anyhow::Result<()> { async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Result<()> { match inception_args.command { - InceptionSubcommands::Ecosystem(args) => commands::ecosystem::run(shell, args).await?, - InceptionSubcommands::Chain(args) => commands::chain::run(shell, args).await?, + InceptionSubcommands::Ecosystem(args) => commands::ecosystem::run(shell, *args).await?, + InceptionSubcommands::Chain(args) => commands::chain::run(shell, *args).await?, InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers(args) => commands::containers::run(shell, args)?, diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 621441ae8d49..ebdcf7378a44 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -72,6 +72,10 @@ pub(super) const MSG_CHAIN_NOT_FOUND_ERR: &str = "Chain not found"; pub(super) const MSG_INITIALIZING_ECOSYSTEM: &str = "Initializing ecosystem"; pub(super) const MSG_DEPLOYING_ERC20: &str = "Deploying ERC20 contracts"; pub(super) const MSG_CHAIN_INITIALIZED: &str = "Chain initialized successfully"; +pub(super) const MSG_CHAIN_CONFIGS_INITIALIZED: &str = "Chain configs were initialized"; +pub(super) const MSG_CHAIN_OWNERSHIP_TRANSFERRED: &str = + "Chain ownership was transferred successfully"; +pub(super) const MSG_CHAIN_REGISTERED: &str = "Chain registraion was successful"; pub(super) const MSG_DISTRIBUTING_ETH_SPINNER: &str = "Distributing eth..."; pub(super) const MSG_MINT_BASE_TOKEN_SPINNER: &str = "Minting base token to the governance addresses..."; @@ -100,7 +104,11 @@ pub(super) fn msg_initializing_chain(chain_name: &str) -> String { } pub(super) fn msg_ecosystem_initialized(chains: &str) -> String { - format!("Ecosystem initialized successfully with chains {chains}") + if chains.is_empty() { + "Ecosystem initialized successfully. You can initialize chain with `chain init`".to_string() + } else { + format!("Ecosystem initialized successfully with chains {chains}") + } } /// Ecosystem default related messages @@ -187,6 +195,7 @@ pub(super) const MSG_INITIALIZING_SERVER_DATABASE: &str = "Initializing server d pub(super) const MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR: &str = "Failed to drop server database"; pub(super) const MSG_INITIALIZING_PROVER_DATABASE: &str = "Initializing prover database"; pub(super) const MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR: &str = "Failed to drop prover database"; +pub(super) const MSG_GENESIS_DATABASES_INITIALIZED: &str = "Databases initialized successfully"; /// Chain update related messages pub(super) const MSG_WALLETS_CONFIG_MUST_BE_PRESENT: &str = "Wallets configuration must be present"; From b7ba4286ce38a2e833087c0b91c584de0f956fa7 Mon Sep 17 00:00:00 2001 From: Danil Date: Wed, 2 Oct 2024 21:05:10 +0200 Subject: [PATCH 4/8] fix(zk_toolbox): Correct secrets (#3004) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`. Signed-off-by: Danil --- zk_toolbox/crates/zk_inception/src/commands/update.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/update.rs b/zk_toolbox/crates/zk_inception/src/commands/update.rs index 5cb7208ffd0c..534d490e6cae 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/update.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/update.rs @@ -8,11 +8,10 @@ use common::{ yaml::{merge_yaml, ConfigDiff}, }; use config::{ - traits::ReadConfigWithBasePath, ChainConfig, EcosystemConfig, CONTRACTS_FILE, EN_CONFIG_FILE, - ERA_OBSERBAVILITY_DIR, GENERAL_FILE, GENESIS_FILE, SECRETS_FILE, + ChainConfig, EcosystemConfig, CONTRACTS_FILE, EN_CONFIG_FILE, ERA_OBSERBAVILITY_DIR, + GENERAL_FILE, GENESIS_FILE, SECRETS_FILE, }; use xshell::Shell; -use zksync_config::configs::Secrets; use super::args::UpdateArgs; use crate::{ @@ -183,7 +182,7 @@ async fn update_chain( )?; } - let secrets = Secrets::read_with_base_path(shell, secrets)?; + let secrets = chain.get_secrets_config()?; if let Some(db) = secrets.database { if let Some(url) = db.server_url { From a5ac9c164bc0d16a879bef1cab1ccdaeac205bdc Mon Sep 17 00:00:00 2001 From: Artem Makhortov <13339874+artmakh@users.noreply.github.com> Date: Thu, 3 Oct 2024 17:22:37 +0700 Subject: [PATCH 5/8] feat(ci): Add external-node build to Makefile (#3002) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Add external-node build to Makefile ## Why ❔ ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`. --- docker/Makefile | 8 ++++++-- docs/guides/build-docker.md | 2 ++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 444a94ce2214..c469587c8ffd 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -56,7 +56,7 @@ check-tools: check-nodejs check-yarn check-rust check-sqlx-cli check-docker chec # Check that contracts are checkout properly check-contracts: @if [ ! -d ../contracts/l1-contracts/lib/forge-std/foundry.toml ] || [ -z "$$(ls -A ../contracts/l1-contracts/lib/forge-std/foundry.toml)" ]; then \ - echo "l1-contracts git submodule is missing. Please re-download repo with `git clone --recurse-submodules https://github.com/matter-labs/zksync-era.git`"; \ + echo "l1-contracts git submodule is missing. Please re-download repo with 'git clone --recurse-submodules https://github.com/matter-labs/zksync-era.git'"; \ exit 1; \ fi @@ -93,9 +93,12 @@ build-witness-generator: check-tools prepare-keys $(DOCKER_BUILD_CMD) --file witness-generator/Dockerfile --load \ --tag witness-generator:$(PROTOCOL_VERSION) $(CONTEXT) +build-external-node: check-tools prepare-contracts + $(DOCKER_BUILD_CMD) --file external-node/Dockerfile --load \ + --tag external-node:$(PROTOCOL_VERSION) $(CONTEXT) # Build all containers -build-all: build-contract-verifier build-server-v2 build-witness-generator build-circuit-prover-gpu cleanup +build-all: build-contract-verifier build-server-v2 build-witness-generator build-circuit-prover-gpu build-external-node cleanup # Clean generated images clean-all: @@ -104,3 +107,4 @@ clean-all: docker rmi server-v2:$(PROTOCOL_VERSION) >/dev/null 2>&1 docker rmi prover:$(PROTOCOL_VERSION) >/dev/null 2>&1 docker rmi witness-generator:$(PROTOCOL_VERSION) >/dev/null 2>&1 + docker rmi external-node:$(PROTOCOL_VERSION) >/dev/null 2>&1 diff --git a/docs/guides/build-docker.md b/docs/guides/build-docker.md index a9e8f5d3e76c..5dd9cff022b9 100644 --- a/docs/guides/build-docker.md +++ b/docs/guides/build-docker.md @@ -25,6 +25,7 @@ contract-verifier:2.0 server-v2:2.0 prover:2.0 witness-generator:2.0 +external-node:2.0 ``` Alternatively, you may build only needed components - available targets are @@ -34,6 +35,7 @@ make -C ./docker build-contract-verifier make -C ./docker build-server-v2 make -C ./docker build-circuit-prover-gpu make -C ./docker build-witness-generator +make -C ./docker build-external-node ``` ## Building updated images From 092eed98751f81a347854a993fdb6913fd7fab2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Fri, 4 Oct 2024 03:17:35 -0300 Subject: [PATCH 6/8] fix(zk_toolbox): Remove prover db from server init (#3009) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Remove prover db from server init ## Why ❔ prover db is not needed in server init since prover db is initialized with prover init --- .github/workflows/ci-core-reusable.yml | 8 ---- zk_toolbox/crates/config/src/secrets.rs | 8 ++-- .../src/commands/chain/args/genesis.rs | 44 ++----------------- .../src/commands/chain/genesis/database.rs | 36 ++++----------- .../src/commands/chain/genesis/mod.rs | 6 +-- 5 files changed, 17 insertions(+), 85 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 9dbd4202afd3..d03e44f8bca4 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -216,8 +216,6 @@ jobs: --deploy-ecosystem --l1-rpc-url=http://localhost:8545 \ --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ --server-db-name=zksync_server_localhost_era \ - --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ - --prover-db-name=zksync_prover_localhost_era \ --ignore-prerequisites --verbose \ --observability=false @@ -246,8 +244,6 @@ jobs: --l1-rpc-url=http://localhost:8545 \ --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ --server-db-name=zksync_server_localhost_validium \ - --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ - --prover-db-name=zksync_prover_localhost_validium \ --chain validium - name: Create and initialize chain with Custom Token @@ -269,8 +265,6 @@ jobs: --l1-rpc-url=http://localhost:8545 \ --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ --server-db-name=zksync_server_localhost_custom_token \ - --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ - --prover-db-name=zksync_prover_localhost_custom_token \ --chain custom_token - name: Create and register chain with transactions signed "offline" @@ -327,8 +321,6 @@ jobs: --l1-rpc-url=http://localhost:8545 \ --server-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ --server-db-name=zksync_server_localhost_consensus \ - --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \ - --prover-db-name=zksync_prover_localhost_consensus \ --chain consensus - name: Build test dependencies diff --git a/zk_toolbox/crates/config/src/secrets.rs b/zk_toolbox/crates/config/src/secrets.rs index f0a39148b034..02ace5da88ef 100644 --- a/zk_toolbox/crates/config/src/secrets.rs +++ b/zk_toolbox/crates/config/src/secrets.rs @@ -12,17 +12,15 @@ use crate::{ traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig}, }; -pub fn set_databases( +pub fn set_server_database( secrets: &mut SecretsConfig, server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, ) -> anyhow::Result<()> { let database = secrets .database .as_mut() - .context("Databases must be presented")?; + .context("Server database must be presented")?; database.server_url = Some(SensitiveUrl::from(server_db_config.full_url())); - database.prover_url = Some(SensitiveUrl::from(prover_db_config.full_url())); Ok(()) } @@ -33,7 +31,7 @@ pub fn set_prover_database( let database = secrets .database .as_mut() - .context("Databases must be presented")?; + .context("Prover database must be presented")?; database.prover_url = Some(SensitiveUrl::from(prover_db_config.full_url())); Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs index 21796b3179de..aaf995985a36 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/genesis.rs @@ -7,11 +7,10 @@ use slugify_rs::slugify; use url::Url; use crate::{ - defaults::{generate_db_names, DBNames, DATABASE_PROVER_URL, DATABASE_SERVER_URL}, + defaults::{generate_db_names, DBNames, DATABASE_SERVER_URL}, messages::{ - msg_prover_db_name_prompt, msg_prover_db_url_prompt, msg_server_db_name_prompt, - msg_server_db_url_prompt, MSG_PROVER_DB_NAME_HELP, MSG_PROVER_DB_URL_HELP, - MSG_SERVER_DB_NAME_HELP, MSG_SERVER_DB_URL_HELP, MSG_USE_DEFAULT_DATABASES_HELP, + msg_server_db_name_prompt, msg_server_db_url_prompt, MSG_SERVER_DB_NAME_HELP, + MSG_SERVER_DB_URL_HELP, MSG_USE_DEFAULT_DATABASES_HELP, }, }; @@ -21,10 +20,6 @@ pub struct GenesisArgs { pub server_db_url: Option, #[clap(long, help = MSG_SERVER_DB_NAME_HELP)] pub server_db_name: Option, - #[clap(long, help = MSG_PROVER_DB_URL_HELP)] - pub prover_db_url: Option, - #[clap(long, help = MSG_PROVER_DB_NAME_HELP)] - pub prover_db_name: Option, #[clap(long, short, help = MSG_USE_DEFAULT_DATABASES_HELP)] pub use_default: bool, #[clap(long, short, action)] @@ -33,15 +28,11 @@ pub struct GenesisArgs { impl GenesisArgs { pub fn fill_values_with_prompt(self, config: &ChainConfig) -> GenesisArgsFinal { - let DBNames { - server_name, - prover_name, - } = generate_db_names(config); + let DBNames { server_name, .. } = generate_db_names(config); let chain_name = config.name.clone(); if self.use_default { GenesisArgsFinal { server_db: DatabaseConfig::new(DATABASE_SERVER_URL.clone(), server_name), - prover_db: DatabaseConfig::new(DATABASE_PROVER_URL.clone(), prover_name), dont_drop: self.dont_drop, } } else { @@ -58,22 +49,8 @@ impl GenesisArgs { }), separator = "_" ); - let prover_db_url = self.prover_db_url.unwrap_or_else(|| { - Prompt::new(&msg_prover_db_url_prompt(&chain_name)) - .default(DATABASE_PROVER_URL.as_str()) - .ask() - }); - let prover_db_name = slugify!( - &self.prover_db_name.unwrap_or_else(|| { - Prompt::new(&msg_prover_db_name_prompt(&chain_name)) - .default(&prover_name) - .ask() - }), - separator = "_" - ); GenesisArgsFinal { server_db: DatabaseConfig::new(server_db_url, server_db_name), - prover_db: DatabaseConfig::new(prover_db_url, prover_db_name), dont_drop: self.dont_drop, } } @@ -96,25 +73,13 @@ impl GenesisArgs { (None, None) }; - let (prover_db_url, prover_db_name) = if let Some(db_full_url) = database.prover_url { - let db_config = DatabaseConfig::from_url(db_full_url.expose_url()) - .context("Invalid prover database URL")?; - (Some(db_config.url), Some(db_config.name)) - } else { - (None, None) - }; - self.server_db_url = self.server_db_url.or(server_db_url); self.server_db_name = self.server_db_name.or(server_db_name); - self.prover_db_url = self.prover_db_url.or(prover_db_url); - self.prover_db_name = self.prover_db_name.or(prover_db_name); Ok(self.fill_values_with_prompt(chain_config)) } pub fn reset_db_names(&mut self) { - self.prover_db_name = None; - self.prover_db_url = None; self.server_db_name = None; self.server_db_url = None; } @@ -123,6 +88,5 @@ impl GenesisArgs { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct GenesisArgsFinal { pub server_db: DatabaseConfig, - pub prover_db: DatabaseConfig, pub dont_drop: bool, } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs index bb78979ec38e..edf480946be1 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/database.rs @@ -7,7 +7,7 @@ use common::{ logger, }; use config::{ - override_config, set_databases, set_file_artifacts, set_rocks_db_config, + override_config, set_file_artifacts, set_rocks_db_config, set_server_database, traits::SaveConfigWithBasePath, ChainConfig, EcosystemConfig, FileArtifacts, }; use types::ProverMode; @@ -18,12 +18,11 @@ use crate::{ commands::chain::args::genesis::{GenesisArgs, GenesisArgsFinal}, consts::{ PATH_TO_ONLY_REAL_PROOFS_OVERRIDE_CONFIG, PATH_TO_VALIDIUM_OVERRIDE_CONFIG, - PROVER_MIGRATIONS, SERVER_MIGRATIONS, + SERVER_MIGRATIONS, }, messages::{ - MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR, - MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, MSG_GENESIS_DATABASES_INITIALIZED, - MSG_INITIALIZING_PROVER_DATABASE, MSG_INITIALIZING_SERVER_DATABASE, + MSG_CHAIN_NOT_INITIALIZED, MSG_FAILED_TO_DROP_SERVER_DATABASE_ERR, + MSG_GENESIS_DATABASES_INITIALIZED, MSG_INITIALIZING_SERVER_DATABASE, MSG_RECREATE_ROCKS_DB_ERRROR, }, utils::rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, @@ -37,13 +36,12 @@ pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { let mut secrets = chain_config.get_secrets_config()?; let args = args.fill_values_with_secrets(&chain_config)?; - set_databases(&mut secrets, &args.server_db, &args.prover_db)?; + set_server_database(&mut secrets, &args.server_db)?; secrets.save_with_base_path(shell, &chain_config.configs)?; - initialize_databases( + initialize_server_database( shell, &args.server_db, - &args.prover_db, chain_config.link_to_code.clone(), args.dont_drop, ) @@ -53,10 +51,9 @@ pub async fn run(args: GenesisArgs, shell: &Shell) -> anyhow::Result<()> { Ok(()) } -pub async fn initialize_databases( +pub async fn initialize_server_database( shell: &Shell, server_db_config: &DatabaseConfig, - prover_db_config: &DatabaseConfig, link_to_code: PathBuf, dont_drop: bool, ) -> anyhow::Result<()> { @@ -78,23 +75,6 @@ pub async fn initialize_databases( ) .await?; - if global_config().verbose { - logger::debug(MSG_INITIALIZING_PROVER_DATABASE) - } - if !dont_drop { - drop_db_if_exists(prover_db_config) - .await - .context(MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR)?; - init_db(prover_db_config).await?; - } - let path_to_prover_migration = link_to_code.join(PROVER_MIGRATIONS); - migrate_db( - shell, - path_to_prover_migration, - &prover_db_config.full_url(), - ) - .await?; - Ok(()) } @@ -107,7 +87,7 @@ pub fn update_configs( // Update secrets configs let mut secrets = config.get_secrets_config()?; - set_databases(&mut secrets, &args.server_db, &args.prover_db)?; + set_server_database(&mut secrets, &args.server_db)?; secrets.save_with_base_path(shell, &config.configs)?; // Update general config diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs index 01842c2916ae..c1cc03174aeb 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/genesis/mod.rs @@ -7,7 +7,7 @@ use xshell::Shell; use crate::{ commands::chain::{ args::genesis::{GenesisArgs, GenesisArgsFinal}, - genesis::{self, database::initialize_databases, server::run_server_genesis}, + genesis::{self, database::initialize_server_database, server::run_server_genesis}, }, messages::{ MSG_CHAIN_NOT_INITIALIZED, MSG_GENESIS_COMPLETED, MSG_INITIALIZING_DATABASES_SPINNER, @@ -70,16 +70,14 @@ pub async fn genesis( logger::object_to_string(serde_json::json!({ "chain_config": config, "server_db_config": args.server_db, - "prover_db_config": args.prover_db, })), ); logger::info(MSG_STARTING_GENESIS); let spinner = Spinner::new(MSG_INITIALIZING_DATABASES_SPINNER); - initialize_databases( + initialize_server_database( shell, &args.server_db, - &args.prover_db, config.link_to_code.clone(), args.dont_drop, ) From 2a7e72b08d214472b1b97dcabd1e675ebe722c90 Mon Sep 17 00:00:00 2001 From: Artem Makhortov <13339874+artmakh@users.noreply.github.com> Date: Fri, 4 Oct 2024 16:53:32 +0700 Subject: [PATCH 7/8] fix(ci): Build zk-env with CUDA (#3013) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Ability to build zk-env with cuda not only from push to main branch ## Why ❔ ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk_supervisor fmt` and `zk_supervisor lint`. --- .github/workflows/zk-environment-publish.yml | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/zk-environment-publish.yml b/.github/workflows/zk-environment-publish.yml index 5a08dff178c4..73303d15cb30 100644 --- a/.github/workflows/zk-environment-publish.yml +++ b/.github/workflows/zk-environment-publish.yml @@ -4,6 +4,12 @@ on: # Workflow dispatch, to allow building and pushing new environments. # It will NOT mark them as latest. workflow_dispatch: + inputs: + build_cuda: + description: "Build CUDA images or not" + type: boolean + required: false + default: false push: branches: @@ -202,25 +208,25 @@ jobs: echo "should_run=$changed_files_output" >> "$GITHUB_OUTPUT" - name: Checkout code - if: steps.condition.outputs.should_run == 'true' + if: ${{ (steps.condition.outputs.should_run == 'true') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 with: submodules: "recursive" - name: Log in to US GAR - if: steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' + if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} run: | gcloud auth print-access-token --lifetime=7200 --impersonate-service-account=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com | docker login -u oauth2accesstoken --password-stdin https://us-docker.pkg.dev - name: Log in to Docker Hub - if: steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' + if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Login to GitHub Container Registry - if: steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main' + if: ${{ (steps.condition.outputs.should_run == 'true' && github.event_name == 'push' && github.ref == 'refs/heads/main') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io @@ -228,19 +234,19 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up QEMU - if: steps.condition.outputs.should_run == 'true' + if: ${{ (steps.condition.outputs.should_run == 'true') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: docker/setup-qemu-action@49b3bc8e6bdd4a60e6116a5414239cba5943d3cf # v3.2.0 - name: Set up Docker Buildx - if: steps.condition.outputs.should_run == 'true' + if: ${{ (steps.condition.outputs.should_run == 'true') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: docker/setup-buildx-action@aa33708b10e362ff993539393ff100fa93ed6a27 # v3.5.0 - name: Build and optionally push - if: steps.condition.outputs.should_run == 'true' + if: ${{ (steps.condition.outputs.should_run == 'true') || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} uses: docker/build-push-action@5176d81f87c23d6fc96624dfdbcd9f3830bbe445 # v6.5.0 with: file: docker/zk-environment/20.04_amd64_cuda_${{ matrix.cuda_version }}.Dockerfile - push: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + push: ${{ ( github.event_name == 'push' && github.ref == 'refs/heads/main' ) || (github.event_name == 'workflow_dispatch' && inputs.build_cuda) }} tags: | us-docker.pkg.dev/matterlabs-infra/matterlabs-docker/zk-environment-cuda-${{ matrix.cuda_version }}:latest matterlabs/zk-environment:cuda-${{ matrix.cuda_version }}-latest From e984bfb8a243bc746549ab9347dc0a367fe02790 Mon Sep 17 00:00:00 2001 From: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Date: Mon, 7 Oct 2024 10:02:09 +0300 Subject: [PATCH 8/8] feat(en): periodically fetch bridge addresses (#2949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Periodically fetch bridge addresses ## Why ❔ Addresses will be changed during gateway upgrade ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- .gitignore | 1 + core/bin/external_node/src/config/mod.rs | 8 ++ core/bin/external_node/src/node_builder.rs | 4 + core/lib/config/src/configs/en_config.rs | 3 +- core/lib/config/src/testonly.rs | 1 + core/lib/protobuf_config/src/en.rs | 11 ++- .../protobuf_config/src/proto/config/en.proto | 1 + .../web3/backend_jsonrpsee/namespaces/zks.rs | 2 +- core/node/api_server/src/web3/mod.rs | 80 +++++++++---------- .../api_server/src/web3/namespaces/zks.rs | 4 +- core/node/api_server/src/web3/state.rs | 72 +++++++---------- core/node/api_server/src/web3/testonly.rs | 4 + .../web3_api/server/bridge_addresses.rs | 48 +++++++++++ .../web3_api/{server.rs => server/mod.rs} | 71 ++++++++++++---- .../layers/web3_api/server/sealed_l2_block.rs | 50 ++++++++++++ .../commands/external_node/prepare_configs.rs | 1 + 16 files changed, 254 insertions(+), 107 deletions(-) create mode 100644 core/node/node_framework/src/implementations/layers/web3_api/server/bridge_addresses.rs rename core/node/node_framework/src/implementations/layers/web3_api/{server.rs => server/mod.rs} (81%) create mode 100644 core/node/node_framework/src/implementations/layers/web3_api/server/sealed_l2_block.rs diff --git a/.gitignore b/.gitignore index c3de7a2df84d..bbd13e2319af 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ prover/data/keys/setup_* # Zk Toolbox chains/era/configs/* +chains/gateway/* configs/* era-observability/ core/tests/ts-integration/deployments-zk diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 9b1677c47c4d..56ee3edfd253 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -445,6 +445,8 @@ pub(crate) struct OptionalENConfig { /// Gateway RPC URL, needed for operating during migration. #[allow(dead_code)] pub gateway_url: Option, + /// Interval for bridge addresses refreshing in seconds. + bridge_addresses_refresh_interval_sec: Option, } impl OptionalENConfig { @@ -675,6 +677,7 @@ impl OptionalENConfig { api_namespaces, contracts_diamond_proxy_addr: None, gateway_url: enconfig.gateway_url.clone(), + bridge_addresses_refresh_interval_sec: enconfig.bridge_addresses_refresh_interval_sec, }) } @@ -901,6 +904,11 @@ impl OptionalENConfig { Duration::from_secs(self.pruning_data_retention_sec) } + pub fn bridge_addresses_refresh_interval(&self) -> Option { + self.bridge_addresses_refresh_interval_sec + .map(|n| Duration::from_secs(n.get())) + } + #[cfg(test)] fn mock() -> Self { // Set all values to their defaults diff --git a/core/bin/external_node/src/node_builder.rs b/core/bin/external_node/src/node_builder.rs index d0055896d42e..14e09b9c2a7a 100644 --- a/core/bin/external_node/src/node_builder.rs +++ b/core/bin/external_node/src/node_builder.rs @@ -430,6 +430,10 @@ impl ExternalNodeBuilder { response_body_size_limit: Some(self.config.optional.max_response_body_size()), with_extended_tracing: self.config.optional.extended_rpc_tracing, pruning_info_refresh_interval: Some(pruning_info_refresh_interval), + bridge_addresses_refresh_interval: self + .config + .optional + .bridge_addresses_refresh_interval(), polling_interval: Some(self.config.optional.polling_interval()), websocket_requests_per_minute_limit: None, // To be set by WS server layer method if required. replication_lag_limit: None, // TODO: Support replication lag limit diff --git a/core/lib/config/src/configs/en_config.rs b/core/lib/config/src/configs/en_config.rs index 7f130e3539a8..4cab47b0779e 100644 --- a/core/lib/config/src/configs/en_config.rs +++ b/core/lib/config/src/configs/en_config.rs @@ -1,4 +1,4 @@ -use std::num::NonZeroUsize; +use std::num::{NonZeroU64, NonZeroUsize}; use serde::Deserialize; use zksync_basic_types::{ @@ -19,4 +19,5 @@ pub struct ENConfig { pub main_node_rate_limit_rps: Option, pub gateway_url: Option, + pub bridge_addresses_refresh_interval_sec: Option, } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 6fbbad9d8ff2..86d9545b0fb4 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -933,6 +933,7 @@ impl Distribution for EncodeDist { main_node_rate_limit_rps: self.sample_opt(|| rng.gen()), gateway_url: self .sample_opt(|| format!("localhost:{}", rng.gen::()).parse().unwrap()), + bridge_addresses_refresh_interval_sec: self.sample_opt(|| rng.gen()), } } } diff --git a/core/lib/protobuf_config/src/en.rs b/core/lib/protobuf_config/src/en.rs index 9c07d1d39297..9d1a39310604 100644 --- a/core/lib/protobuf_config/src/en.rs +++ b/core/lib/protobuf_config/src/en.rs @@ -1,4 +1,7 @@ -use std::{num::NonZeroUsize, str::FromStr}; +use std::{ + num::{NonZeroU64, NonZeroUsize}, + str::FromStr, +}; use anyhow::Context; use zksync_basic_types::{url::SensitiveUrl, L1ChainId, L2ChainId}; @@ -36,6 +39,9 @@ impl ProtoRepr for proto::ExternalNode { .as_ref() .map(|a| a.parse().context("gateway_url")) .transpose()?, + bridge_addresses_refresh_interval_sec: self + .bridge_addresses_refresh_interval_sec + .and_then(NonZeroU64::new), }) } @@ -55,6 +61,9 @@ impl ProtoRepr for proto::ExternalNode { .gateway_url .as_ref() .map(|a| a.expose_str().to_string()), + bridge_addresses_refresh_interval_sec: this + .bridge_addresses_refresh_interval_sec + .map(|a| a.get()), } } } diff --git a/core/lib/protobuf_config/src/proto/config/en.proto b/core/lib/protobuf_config/src/proto/config/en.proto index d8a13d31d4b9..69412704ea0f 100644 --- a/core/lib/protobuf_config/src/proto/config/en.proto +++ b/core/lib/protobuf_config/src/proto/config/en.proto @@ -10,4 +10,5 @@ message ExternalNode { optional uint64 main_node_rate_limit_rps = 6; // optional optional config.genesis.L1BatchCommitDataGeneratorMode l1_batch_commit_data_generator_mode = 7; // optional, default to rollup optional string gateway_url = 8; // optional + optional uint64 bridge_addresses_refresh_interval_sec = 9; // optional } diff --git a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs index f83eb37ad962..31c8f15bb1ea 100644 --- a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs +++ b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs @@ -55,7 +55,7 @@ impl ZksNamespaceServer for ZksNamespace { } async fn get_bridge_contracts(&self) -> RpcResult { - Ok(self.get_bridge_contracts_impl()) + Ok(self.get_bridge_contracts_impl().await) } async fn l1_chain_id(&self) -> RpcResult { diff --git a/core/node/api_server/src/web3/mod.rs b/core/node/api_server/src/web3/mod.rs index bad1b493a5fd..620e9185078e 100644 --- a/core/node/api_server/src/web3/mod.rs +++ b/core/node/api_server/src/web3/mod.rs @@ -47,6 +47,7 @@ use self::{ use crate::{ execution_sandbox::{BlockStartInfo, VmConcurrencyBarrier}, tx_sender::TxSender, + web3::state::BridgeAddressesHandle, }; pub mod backend_jsonrpsee; @@ -143,7 +144,6 @@ struct OptionalApiParams { #[derive(Debug)] pub struct ApiServer { pool: ConnectionPool, - updaters_pool: ConnectionPool, health_updater: Arc, config: InternalApiConfig, transport: ApiTransport, @@ -153,18 +153,21 @@ pub struct ApiServer { namespaces: Vec, method_tracer: Arc, optional: OptionalApiParams, + bridge_addresses_handle: BridgeAddressesHandle, + sealed_l2_block_handle: SealedL2BlockNumber, } #[derive(Debug)] pub struct ApiBuilder { pool: ConnectionPool, - updaters_pool: ConnectionPool, config: InternalApiConfig, polling_interval: Duration, pruning_info_refresh_interval: Duration, // Mandatory params that must be set using builder methods. transport: Option, tx_sender: Option, + bridge_addresses_handle: Option, + sealed_l2_block_handle: Option, // Optional params that may or may not be set using builder methods. We treat `namespaces` // specially because we want to output a warning if they are not set. namespaces: Option>, @@ -178,13 +181,14 @@ impl ApiBuilder { pub fn jsonrpsee_backend(config: InternalApiConfig, pool: ConnectionPool) -> Self { Self { - updaters_pool: pool.clone(), pool, config, polling_interval: Self::DEFAULT_POLLING_INTERVAL, pruning_info_refresh_interval: Self::DEFAULT_PRUNING_INFO_REFRESH_INTERVAL, transport: None, tx_sender: None, + bridge_addresses_handle: None, + sealed_l2_block_handle: None, namespaces: None, method_tracer: Arc::new(MethodTracer::default()), optional: OptionalApiParams::default(), @@ -201,15 +205,6 @@ impl ApiBuilder { self } - /// Configures a dedicated DB pool to be used for updating different information, - /// such as last mined block number or account nonces. This pool is used to execute - /// in a background task. If not called, the main pool will be used. If the API server is under high load, - /// it may make sense to supply a single-connection pool to reduce pool contention with the API methods. - pub fn with_updaters_pool(mut self, pool: ConnectionPool) -> Self { - self.updaters_pool = pool; - self - } - pub fn with_tx_sender(mut self, tx_sender: TxSender) -> Self { self.tx_sender = Some(tx_sender); self @@ -285,6 +280,22 @@ impl ApiBuilder { self } + pub fn with_sealed_l2_block_handle( + mut self, + sealed_l2_block_handle: SealedL2BlockNumber, + ) -> Self { + self.sealed_l2_block_handle = Some(sealed_l2_block_handle); + self + } + + pub fn with_bridge_addresses_handle( + mut self, + bridge_addresses_handle: BridgeAddressesHandle, + ) -> Self { + self.bridge_addresses_handle = Some(bridge_addresses_handle); + self + } + // Intended for tests only. #[doc(hidden)] fn with_pub_sub_events(mut self, sender: mpsc::UnboundedSender) -> Self { @@ -312,7 +323,6 @@ impl ApiBuilder { Ok(ApiServer { pool: self.pool, health_updater: Arc::new(health_updater), - updaters_pool: self.updaters_pool, config: self.config, transport, tx_sender: self.tx_sender.context("Transaction sender not set")?, @@ -326,6 +336,12 @@ impl ApiBuilder { }), method_tracer: self.method_tracer, optional: self.optional, + sealed_l2_block_handle: self + .sealed_l2_block_handle + .context("Sealed l2 block handle not set")?, + bridge_addresses_handle: self + .bridge_addresses_handle + .context("Bridge addresses handle not set")?, }) } } @@ -335,11 +351,8 @@ impl ApiServer { self.health_updater.subscribe() } - async fn build_rpc_state( - self, - last_sealed_l2_block: SealedL2BlockNumber, - ) -> anyhow::Result { - let mut storage = self.updaters_pool.connection_tagged("api").await?; + async fn build_rpc_state(self) -> anyhow::Result { + let mut storage = self.pool.connection_tagged("api").await?; let start_info = BlockStartInfo::new(&mut storage, self.pruning_info_refresh_interval).await?; drop(storage); @@ -363,7 +376,8 @@ impl ApiServer { api_config: self.config, start_info, mempool_cache: self.optional.mempool_cache, - last_sealed_l2_block, + last_sealed_l2_block: self.sealed_l2_block_handle, + bridge_addresses_handle: self.bridge_addresses_handle, tree_api: self.optional.tree_api, }) } @@ -371,11 +385,10 @@ impl ApiServer { async fn build_rpc_module( self, pub_sub: Option, - last_sealed_l2_block: SealedL2BlockNumber, ) -> anyhow::Result> { let namespaces = self.namespaces.clone(); let zksync_network_id = self.config.l2_chain_id; - let rpc_state = self.build_rpc_state(last_sealed_l2_block).await?; + let rpc_state = self.build_rpc_state().await?; // Collect all the methods into a single RPC module. let mut rpc = RpcModule::new(()); @@ -473,21 +486,9 @@ impl ApiServer { self, stop_receiver: watch::Receiver, ) -> anyhow::Result { - // Chosen to be significantly smaller than the interval between L2 blocks, but larger than - // the latency of getting the latest sealed L2 block number from Postgres. If the API server - // processes enough requests, information about the latest sealed L2 block will be updated - // by reporting block difference metrics, so the actual update lag would be much smaller than this value. - const SEALED_L2_BLOCK_UPDATE_INTERVAL: Duration = Duration::from_millis(25); - let transport = self.transport; + let mut tasks = vec![]; - let (last_sealed_l2_block, sealed_l2_block_update_task) = SealedL2BlockNumber::new( - self.updaters_pool.clone(), - SEALED_L2_BLOCK_UPDATE_INTERVAL, - stop_receiver.clone(), - ); - - let mut tasks = vec![tokio::spawn(sealed_l2_block_update_task)]; let pub_sub = if matches!(transport, ApiTransport::WebSocket(_)) && self.namespaces.contains(&Namespace::Pubsub) { @@ -510,12 +511,8 @@ impl ApiServer { // framework it'll no longer be needed. let health_check = self.health_updater.subscribe(); let (local_addr_sender, local_addr) = oneshot::channel(); - let server_task = tokio::spawn(self.run_jsonrpsee_server( - stop_receiver, - pub_sub, - last_sealed_l2_block, - local_addr_sender, - )); + let server_task = + tokio::spawn(self.run_jsonrpsee_server(stop_receiver, pub_sub, local_addr_sender)); tasks.push(server_task); Ok(ApiServerHandles { @@ -584,7 +581,6 @@ impl ApiServer { self, mut stop_receiver: watch::Receiver, pub_sub: Option, - last_sealed_l2_block: SealedL2BlockNumber, local_addr_sender: oneshot::Sender, ) -> anyhow::Result<()> { let transport = self.transport; @@ -640,7 +636,7 @@ impl ApiServer { tracing::info!("Enabled extended call tracing for {transport_str} API server; this might negatively affect performance"); } - let rpc = self.build_rpc_module(pub_sub, last_sealed_l2_block).await?; + let rpc = self.build_rpc_module(pub_sub).await?; let registered_method_names = Arc::new(rpc.method_names().collect::>()); tracing::debug!( "Built RPC module for {transport_str} server with {} methods: {registered_method_names:?}", diff --git a/core/node/api_server/src/web3/namespaces/zks.rs b/core/node/api_server/src/web3/namespaces/zks.rs index 61456095d67c..2192f11eb14e 100644 --- a/core/node/api_server/src/web3/namespaces/zks.rs +++ b/core/node/api_server/src/web3/namespaces/zks.rs @@ -132,8 +132,8 @@ impl ZksNamespace { self.state.api_config.l2_testnet_paymaster_addr } - pub fn get_bridge_contracts_impl(&self) -> BridgeAddresses { - self.state.api_config.bridge_addresses.clone() + pub async fn get_bridge_contracts_impl(&self) -> BridgeAddresses { + self.state.bridge_addresses_handle.read().await } pub fn l1_chain_id_impl(&self) -> U64 { diff --git a/core/node/api_server/src/web3/state.rs b/core/node/api_server/src/web3/state.rs index 8cbb75103cd9..723661ab9087 100644 --- a/core/node/api_server/src/web3/state.rs +++ b/core/node/api_server/src/web3/state.rs @@ -4,13 +4,13 @@ use std::{ atomic::{AtomicU32, Ordering}, Arc, }, - time::{Duration, Instant}, + time::Instant, }; use anyhow::Context as _; use futures::TryFutureExt; use lru::LruCache; -use tokio::sync::{watch, Mutex}; +use tokio::sync::{Mutex, RwLock}; use vise::GaugeGuard; use zksync_config::{ configs::{api::Web3JsonRpcConfig, ContractsConfig}, @@ -20,8 +20,9 @@ use zksync_dal::{Connection, ConnectionPool, Core, CoreDal, DalError}; use zksync_metadata_calculator::api_server::TreeApiClient; use zksync_node_sync::SyncState; use zksync_types::{ - api, commitment::L1BatchCommitmentMode, l2::L2Tx, transaction_request::CallRequest, Address, - L1BatchNumber, L1ChainId, L2BlockNumber, L2ChainId, H256, U256, U64, + api, api::BridgeAddresses, commitment::L1BatchCommitmentMode, l2::L2Tx, + transaction_request::CallRequest, Address, L1BatchNumber, L1ChainId, L2BlockNumber, L2ChainId, + H256, U256, U64, }; use zksync_web3_decl::{error::Web3Error, types::Filter}; @@ -173,51 +174,16 @@ impl InternalApiConfig { /// Thread-safe updatable information about the last sealed L2 block number. /// /// The information may be temporarily outdated and thus should only be used where this is OK -/// (e.g., for metrics reporting). The value is updated by [`Self::diff()`] and [`Self::diff_with_block_args()`] -/// and on an interval specified when creating an instance. -#[derive(Debug, Clone)] -pub(crate) struct SealedL2BlockNumber(Arc); +/// (e.g., for metrics reporting). The value is updated by [`Self::diff()`] and [`Self::diff_with_block_args()`]. +#[derive(Debug, Clone, Default)] +pub struct SealedL2BlockNumber(Arc); impl SealedL2BlockNumber { - /// Creates a handle to the last sealed L2 block number together with a task that will update - /// it on a schedule. - pub fn new( - connection_pool: ConnectionPool, - update_interval: Duration, - stop_receiver: watch::Receiver, - ) -> (Self, impl Future>) { - let this = Self(Arc::default()); - let number_updater = this.clone(); - - let update_task = async move { - loop { - if *stop_receiver.borrow() { - tracing::debug!("Stopping latest sealed L2 block updates"); - return Ok(()); - } - - let mut connection = connection_pool.connection_tagged("api").await.unwrap(); - let Some(last_sealed_l2_block) = - connection.blocks_dal().get_sealed_l2_block_number().await? - else { - tokio::time::sleep(update_interval).await; - continue; - }; - drop(connection); - - number_updater.update(last_sealed_l2_block); - tokio::time::sleep(update_interval).await; - } - }; - - (this, update_task) - } - /// Potentially updates the last sealed L2 block number by comparing it to the provided /// sealed L2 block number (not necessarily the last one). /// /// Returns the last sealed L2 block number after the update. - fn update(&self, maybe_newer_l2_block_number: L2BlockNumber) -> L2BlockNumber { + pub fn update(&self, maybe_newer_l2_block_number: L2BlockNumber) -> L2BlockNumber { let prev_value = self .0 .fetch_max(maybe_newer_l2_block_number.0, Ordering::Relaxed); @@ -231,7 +197,7 @@ impl SealedL2BlockNumber { /// Returns the difference between the latest L2 block number and the resolved L2 block number /// from `block_args`. - pub fn diff_with_block_args(&self, block_args: &BlockArgs) -> u32 { + pub(crate) fn diff_with_block_args(&self, block_args: &BlockArgs) -> u32 { // We compute the difference in any case, since it may update the stored value. let diff = self.diff(block_args.resolved_block_number()); @@ -243,6 +209,23 @@ impl SealedL2BlockNumber { } } +#[derive(Debug, Clone)] +pub struct BridgeAddressesHandle(Arc>); + +impl BridgeAddressesHandle { + pub fn new(bridge_addresses: BridgeAddresses) -> Self { + Self(Arc::new(RwLock::new(bridge_addresses))) + } + + pub async fn update(&self, bridge_addresses: BridgeAddresses) { + *self.0.write().await = bridge_addresses; + } + + pub async fn read(&self) -> BridgeAddresses { + self.0.read().await.clone() + } +} + /// Holder for the data required for the API to be functional. #[derive(Debug, Clone)] pub(crate) struct RpcState { @@ -258,6 +241,7 @@ pub(crate) struct RpcState { pub(super) start_info: BlockStartInfo, pub(super) mempool_cache: Option, pub(super) last_sealed_l2_block: SealedL2BlockNumber, + pub(super) bridge_addresses_handle: BridgeAddressesHandle, } impl RpcState { diff --git a/core/node/api_server/src/web3/testonly.rs b/core/node/api_server/src/web3/testonly.rs index 93309fc09cf1..3b05e235c6d4 100644 --- a/core/node/api_server/src/web3/testonly.rs +++ b/core/node/api_server/src/web3/testonly.rs @@ -181,6 +181,8 @@ async fn spawn_server( let mut namespaces = Namespace::DEFAULT.to_vec(); namespaces.extend([Namespace::Debug, Namespace::Snapshots, Namespace::Unstable]); + let sealed_l2_block_handle = SealedL2BlockNumber::default(); + let bridge_addresses_handle = BridgeAddressesHandle::new(api_config.bridge_addresses.clone()); let server_builder = match transport { ApiTransportLabel::Http => ApiBuilder::jsonrpsee_backend(api_config, pool).http(0), @@ -202,6 +204,8 @@ async fn spawn_server( .with_pub_sub_events(pub_sub_events_sender) .with_method_tracer(method_tracer) .enable_api_namespaces(namespaces) + .with_sealed_l2_block_handle(sealed_l2_block_handle) + .with_bridge_addresses_handle(bridge_addresses_handle) .build() .expect("Unable to build API server") .run(stop_receiver) diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server/bridge_addresses.rs b/core/node/node_framework/src/implementations/layers/web3_api/server/bridge_addresses.rs new file mode 100644 index 000000000000..4ba8098c8399 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/web3_api/server/bridge_addresses.rs @@ -0,0 +1,48 @@ +use std::time::Duration; + +use zksync_node_api_server::web3::state::BridgeAddressesHandle; +use zksync_web3_decl::{ + client::{DynClient, L2}, + namespaces::ZksNamespaceClient, +}; + +use crate::{StopReceiver, Task, TaskId}; + +#[derive(Debug)] +pub struct BridgeAddressesUpdaterTask { + pub bridge_address_updater: BridgeAddressesHandle, + pub main_node_client: Box>, + pub update_interval: Option, +} + +#[async_trait::async_trait] +impl Task for BridgeAddressesUpdaterTask { + fn id(&self) -> TaskId { + "bridge_addresses_updater_task".into() + } + + async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + const DEFAULT_INTERVAL: Duration = Duration::from_secs(30); + + let update_interval = self.update_interval.unwrap_or(DEFAULT_INTERVAL); + while !*stop_receiver.0.borrow_and_update() { + match self.main_node_client.get_bridge_contracts().await { + Ok(bridge_addresses) => { + self.bridge_address_updater.update(bridge_addresses).await; + } + Err(err) => { + tracing::error!("Failed to query `get_bridge_contracts`, error: {err:?}"); + } + } + + if tokio::time::timeout(update_interval, stop_receiver.0.changed()) + .await + .is_ok() + { + break; + } + } + + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server.rs b/core/node/node_framework/src/implementations/layers/web3_api/server/mod.rs similarity index 81% rename from core/node/node_framework/src/implementations/layers/web3_api/server.rs rename to core/node/node_framework/src/implementations/layers/web3_api/server/mod.rs index 0a39ae747c71..390d321647cf 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/server.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/server/mod.rs @@ -3,15 +3,24 @@ use std::{num::NonZeroU32, time::Duration}; use tokio::{sync::oneshot, task::JoinHandle}; use zksync_circuit_breaker::replication_lag::ReplicationLagChecker; use zksync_config::configs::api::MaxResponseSize; -use zksync_node_api_server::web3::{state::InternalApiConfig, ApiBuilder, ApiServer, Namespace}; +use zksync_node_api_server::web3::{ + state::{BridgeAddressesHandle, InternalApiConfig, SealedL2BlockNumber}, + ApiBuilder, ApiServer, Namespace, +}; use crate::{ - implementations::resources::{ - circuit_breakers::CircuitBreakersResource, - healthcheck::AppHealthCheckResource, - pools::{PoolResource, ReplicaPool}, - sync_state::SyncStateResource, - web3_api::{MempoolCacheResource, TreeApiClientResource, TxSenderResource}, + implementations::{ + layers::web3_api::server::{ + bridge_addresses::BridgeAddressesUpdaterTask, sealed_l2_block::SealedL2BlockUpdaterTask, + }, + resources::{ + circuit_breakers::CircuitBreakersResource, + healthcheck::AppHealthCheckResource, + main_node_client::MainNodeClientResource, + pools::{PoolResource, ReplicaPool}, + sync_state::SyncStateResource, + web3_api::{MempoolCacheResource, TreeApiClientResource, TxSenderResource}, + }, }, service::StopReceiver, task::{Task, TaskId}, @@ -19,6 +28,9 @@ use crate::{ FromContext, IntoContext, }; +mod bridge_addresses; +mod sealed_l2_block; + /// Set of optional variables that can be altered to modify the behavior of API builder. #[derive(Debug, Default)] pub struct Web3ServerOptionalConfig { @@ -33,6 +45,8 @@ pub struct Web3ServerOptionalConfig { pub replication_lag_limit: Option, // Used by the external node. pub pruning_info_refresh_interval: Option, + // Used by the external node. + pub bridge_addresses_refresh_interval: Option, pub polling_interval: Option, } @@ -61,6 +75,10 @@ impl Web3ServerOptionalConfig { if let Some(polling_interval) = self.polling_interval { api_builder = api_builder.with_polling_interval(polling_interval); } + if let Some(pruning_info_refresh_interval) = self.pruning_info_refresh_interval { + api_builder = + api_builder.with_pruning_info_refresh_interval(pruning_info_refresh_interval); + } api_builder = api_builder.with_extended_tracing(self.with_extended_tracing); api_builder } @@ -109,6 +127,7 @@ pub struct Input { pub circuit_breakers: CircuitBreakersResource, #[context(default)] pub app_health: AppHealthCheckResource, + pub main_node_client: Option, } #[derive(Debug, IntoContext)] @@ -118,6 +137,10 @@ pub struct Output { pub web3_api_task: Web3ApiTask, #[context(task)] pub garbage_collector_task: ApiTaskGarbageCollector, + #[context(task)] + pub sealed_l2_block_updater_task: SealedL2BlockUpdaterTask, + #[context(task)] + pub bridge_addresses_updater_task: Option, } impl Web3ServerLayer { @@ -163,20 +186,39 @@ impl WiringLayer for Web3ServerLayer { async fn wire(self, input: Self::Input) -> Result { // Get required resources. let replica_resource_pool = input.replica_pool; - let updaters_pool = replica_resource_pool.get_custom(2).await?; + let updaters_pool = replica_resource_pool.get_custom(1).await?; let replica_pool = replica_resource_pool.get().await?; let TxSenderResource(tx_sender) = input.tx_sender; let MempoolCacheResource(mempool_cache) = input.mempool_cache; let sync_state = input.sync_state.map(|state| state.0); let tree_api_client = input.tree_api_client.map(|client| client.0); + let sealed_l2_block_handle = SealedL2BlockNumber::default(); + let bridge_addresses_handle = + BridgeAddressesHandle::new(self.internal_api_config.bridge_addresses.clone()); + + let sealed_l2_block_updater_task = SealedL2BlockUpdaterTask { + number_updater: sealed_l2_block_handle.clone(), + pool: updaters_pool, + }; + // Bridge addresses updater task must be started for ENs and only for ENs. + let bridge_addresses_updater_task = + input + .main_node_client + .map(|main_node_client| BridgeAddressesUpdaterTask { + bridge_address_updater: bridge_addresses_handle.clone(), + main_node_client: main_node_client.0, + update_interval: self.optional_config.bridge_addresses_refresh_interval, + }); + // Build server. let mut api_builder = ApiBuilder::jsonrpsee_backend(self.internal_api_config, replica_pool.clone()) - .with_updaters_pool(updaters_pool) .with_tx_sender(tx_sender) .with_mempool_cache(mempool_cache) - .with_extended_tracing(self.optional_config.with_extended_tracing); + .with_extended_tracing(self.optional_config.with_extended_tracing) + .with_sealed_l2_block_handle(sealed_l2_block_handle) + .with_bridge_addresses_handle(bridge_addresses_handle); if let Some(client) = tree_api_client { api_builder = api_builder.with_tree_api(client); } @@ -191,14 +233,9 @@ impl WiringLayer for Web3ServerLayer { if let Some(sync_state) = sync_state { api_builder = api_builder.with_sync_state(sync_state); } - if let Some(pruning_info_refresh_interval) = - self.optional_config.pruning_info_refresh_interval - { - api_builder = - api_builder.with_pruning_info_refresh_interval(pruning_info_refresh_interval); - } let replication_lag_limit = self.optional_config.replication_lag_limit; api_builder = self.optional_config.apply(api_builder); + let server = api_builder.build()?; // Insert healthcheck. @@ -230,6 +267,8 @@ impl WiringLayer for Web3ServerLayer { Ok(Output { web3_api_task, garbage_collector_task, + sealed_l2_block_updater_task, + bridge_addresses_updater_task, }) } } diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server/sealed_l2_block.rs b/core/node/node_framework/src/implementations/layers/web3_api/server/sealed_l2_block.rs new file mode 100644 index 000000000000..02552e212cd6 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/web3_api/server/sealed_l2_block.rs @@ -0,0 +1,50 @@ +use std::time::Duration; + +use zksync_dal::{Core, CoreDal}; +use zksync_db_connection::connection_pool::ConnectionPool; +use zksync_node_api_server::web3::state::SealedL2BlockNumber; + +use crate::{StopReceiver, Task, TaskId}; + +#[derive(Debug)] +pub struct SealedL2BlockUpdaterTask { + pub number_updater: SealedL2BlockNumber, + pub pool: ConnectionPool, +} + +#[async_trait::async_trait] +impl Task for SealedL2BlockUpdaterTask { + fn id(&self) -> TaskId { + "api_sealed_l2_block_updater_task".into() + } + + async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + // Chosen to be significantly smaller than the interval between L2 blocks, but larger than + // the latency of getting the latest sealed L2 block number from Postgres. If the API server + // processes enough requests, information about the latest sealed L2 block will be updated + // by reporting block difference metrics, so the actual update lag would be much smaller than this value. + const UPDATE_INTERVAL: Duration = Duration::from_millis(25); + + while !*stop_receiver.0.borrow_and_update() { + let mut connection = self.pool.connection_tagged("api").await.unwrap(); + let Some(last_sealed_l2_block) = + connection.blocks_dal().get_sealed_l2_block_number().await? + else { + tokio::time::sleep(UPDATE_INTERVAL).await; + continue; + }; + drop(connection); + + self.number_updater.update(last_sealed_l2_block); + + if tokio::time::timeout(UPDATE_INTERVAL, stop_receiver.0.changed()) + .await + .is_ok() + { + break; + } + } + + Ok(()) + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index defbbd12d401..5ab859d17f0a 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -76,6 +76,7 @@ fn prepare_configs( )?, main_node_rate_limit_rps: None, gateway_url: None, + bridge_addresses_refresh_interval_sec: None, }; let mut general_en = general.clone();