From 2f6cd41642d9c2680f17e5c1adf22ad8e1b0288a Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 17 Jun 2024 10:52:11 +0300 Subject: [PATCH 01/43] fix(object-store): Consider more GCS errors transient (#2246) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Considers `reqwest::Error`s with "request" kind transient for the object store logic. ## Why ❔ Similar to some other kinds of `reqwest::Error`s, not all "request" errors are truly transient, but considering them as such is a safer option. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- core/lib/object_store/src/gcs.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/core/lib/object_store/src/gcs.rs b/core/lib/object_store/src/gcs.rs index 6960bd51f2fb..2d4fae77ab80 100644 --- a/core/lib/object_store/src/gcs.rs +++ b/core/lib/object_store/src/gcs.rs @@ -87,7 +87,7 @@ impl From for ObjectStoreError { fn from(err: AuthError) -> Self { let is_transient = matches!( &err, - AuthError::HttpError(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err) || upstream_unavailable(err) + AuthError::HttpError(err) if is_transient_http_error(err) ); Self::Initialization { source: err.into(), @@ -96,6 +96,17 @@ impl From for ObjectStoreError { } } +fn is_transient_http_error(err: &reqwest::Error) -> bool { + err.is_timeout() + || err.is_connect() + // Not all request errors are logically transient, but a significant part of them are (e.g., + // `hyper` protocol-level errors), and it's safer to consider an error transient. + || err.is_request() + || has_transient_io_source(err) + || err.status() == Some(StatusCode::BAD_GATEWAY) + || err.status() == Some(StatusCode::SERVICE_UNAVAILABLE) +} + fn has_transient_io_source(mut err: &(dyn StdError + 'static)) -> bool { loop { if err.is::() { @@ -111,11 +122,6 @@ fn has_transient_io_source(mut err: &(dyn StdError + 'static)) -> bool { } } -fn upstream_unavailable(err: &reqwest::Error) -> bool { - err.status() == Some(StatusCode::BAD_GATEWAY) - || err.status() == Some(StatusCode::SERVICE_UNAVAILABLE) -} - impl From for ObjectStoreError { fn from(err: HttpError) -> Self { let is_not_found = match &err { @@ -131,7 +137,7 @@ impl From for ObjectStoreError { } else { let is_transient = matches!( &err, - HttpError::HttpClient(err) if err.is_timeout() || err.is_connect() || has_transient_io_source(err) || upstream_unavailable(err) + HttpError::HttpClient(err) if is_transient_http_error(err) ); ObjectStoreError::Other { is_transient, From 3f521ace420d3f65e5612c2b6baf096c391ffd7c Mon Sep 17 00:00:00 2001 From: Agustin Aon <21188659+aon@users.noreply.github.com> Date: Mon, 17 Jun 2024 04:52:41 -0400 Subject: [PATCH 02/43] fix: zk-toolbox integration tests ci (#2226) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ - Improves zk-toolbox integration tests CI adding missing arguments from typescript core CI ## Why ❔ ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- .github/workflows/ci-zk-toolbox-reusable.yml | 4 +++- .../crates/zk_inception/src/commands/args/run_server.rs | 7 ++++++- zk_toolbox/crates/zk_inception/src/messages.rs | 2 ++ 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index c3ef46453f13..f3238566eeec 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -28,6 +28,7 @@ jobs: - name: Start services run: | ci_localnet_up + ci_run sccache --start-server - name: Build run: | @@ -75,6 +76,7 @@ jobs: - name: Start services run: | ci_localnet_up + ci_run sccache --start-server - name: Initialize ecosystem run: | @@ -88,7 +90,7 @@ jobs: - name: Run server run: | - ci_run zk_inception server --ignore-prerequisites &>server.log & + ci_run zk_inception server --ignore-prerequisites -a --use-node-framework --verbose &>server.log & ci_run sleep 5 - name: Run integration tests diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs index 47ab8dc75c5d..1ec211c25f6d 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/run_server.rs @@ -1,7 +1,9 @@ use clap::Parser; use serde::{Deserialize, Serialize}; -use crate::messages::{MSG_SERVER_COMPONENTS_HELP, MSG_SERVER_GENESIS_HELP}; +use crate::messages::{ + MSG_SERVER_ADDITIONAL_ARGS_HELP, MSG_SERVER_COMPONENTS_HELP, MSG_SERVER_GENESIS_HELP, +}; #[derive(Debug, Serialize, Deserialize, Parser)] pub struct RunServerArgs { @@ -9,4 +11,7 @@ pub struct RunServerArgs { pub components: Option>, #[clap(long, help = MSG_SERVER_GENESIS_HELP)] pub genesis: bool, + #[clap(long, short)] + #[arg(trailing_var_arg = true, allow_hyphen_values = true, hide = false, help = MSG_SERVER_ADDITIONAL_ARGS_HELP)] + additional_args: Vec, } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 7221f030d415..21f051470555 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -148,6 +148,8 @@ pub(super) const MSG_DEPLOYING_PAYMASTER: &str = "Deploying paymaster"; /// Run server related messages pub(super) const MSG_SERVER_COMPONENTS_HELP: &str = "Components of server to run"; pub(super) const MSG_SERVER_GENESIS_HELP: &str = "Run server in genesis mode"; +pub(super) const MSG_SERVER_ADDITIONAL_ARGS_HELP: &str = + "Additional arguments that can be passed through the CLI"; /// Accept ownership related messages pub(super) const MSG_ACCEPTING_GOVERNANCE_SPINNER: &str = "Accepting governance..."; From d71287402b95cfa974078f1f43e52082f866d287 Mon Sep 17 00:00:00 2001 From: Daniyar Itegulov Date: Mon, 17 Jun 2024 22:29:11 +1000 Subject: [PATCH 03/43] chore: lower function selector log level (#2251) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Lowers function selector not found log level to debug ## Why ❔ It's non-actionable, harmless and spammy at the same time, for example: ![Screenshot 2024-06-17 at 5 10 15 PM](https://github.com/matter-labs/zksync-era/assets/7879134/90b831e1-6391-43da-94da-444a2b47030b) ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- core/lib/multivm/src/interface/types/errors/vm_revert_reason.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/multivm/src/interface/types/errors/vm_revert_reason.rs b/core/lib/multivm/src/interface/types/errors/vm_revert_reason.rs index 25b394ce2582..6b211d543a92 100644 --- a/core/lib/multivm/src/interface/types/errors/vm_revert_reason.rs +++ b/core/lib/multivm/src/interface/types/errors/vm_revert_reason.rs @@ -111,7 +111,7 @@ impl VmRevertReason { function_selector: function_selector.to_vec(), data: bytes.to_vec(), }; - tracing::warn!("Unsupported error type: {}", result); + tracing::debug!("Unsupported error type: {}", result); Ok(result) } } From 3cad74e1cb8452e270fe50df33b07d14dc24a71c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Mon, 17 Jun 2024 17:15:43 +0200 Subject: [PATCH 04/43] feat(ci): add retried rust toolchain installatoin (#2249) Signed-off-by: tomg10 --- .github/workflows/ci-common-reusable.yml | 1 + .github/workflows/ci-core-lint-reusable.yml | 1 + .github/workflows/ci-core-reusable.yml | 3 +++ .github/workflows/ci-prover-reusable.yml | 1 + 4 files changed, 6 insertions(+) diff --git a/.github/workflows/ci-common-reusable.yml b/.github/workflows/ci-common-reusable.yml index 98b7d7ea1a02..191c69180631 100644 --- a/.github/workflows/ci-common-reusable.yml +++ b/.github/workflows/ci-common-reusable.yml @@ -29,6 +29,7 @@ jobs: - name: Init run: | ci_run zk + ci_run run_retried rustup show ci_run zk db setup # This does both linting and "building". We're using `zk lint prover` as it's common practice within our repo diff --git a/.github/workflows/ci-core-lint-reusable.yml b/.github/workflows/ci-core-lint-reusable.yml index 9ee11016f95a..4b67a8ab5cd2 100644 --- a/.github/workflows/ci-core-lint-reusable.yml +++ b/.github/workflows/ci-core-lint-reusable.yml @@ -28,6 +28,7 @@ jobs: - name: Setup db run: | ci_run zk + ci_run run_retried rustup show ci_run zk db migrate - name: Lints diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index 72e75e085b16..b15bc0c41997 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -53,6 +53,7 @@ jobs: - name: Init run: | ci_run zk + ci_run run_retried rustup show ci_run zk run yarn ci_run zk db setup ci_run zk compiler all @@ -192,6 +193,7 @@ jobs: ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts ci_run git config --global --add safe.directory /usr/src/zksync/contracts ci_run zk + ci_run run_retried rustup show if [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then ci_run zk env dev_validium_docker ci_run zk config compile dev_validium_docker @@ -333,6 +335,7 @@ jobs: ci_run git config --global --add safe.directory /usr/src/zksync/contracts/system-contracts ci_run git config --global --add safe.directory /usr/src/zksync/contracts ci_run zk + ci_run run_retried rustup show if [[ "${{ matrix.deployment_mode }}" == "Rollup" ]]; then ci_run zk config compile elif [[ "${{ matrix.deployment_mode }}" == "Validium" ]]; then diff --git a/.github/workflows/ci-prover-reusable.yml b/.github/workflows/ci-prover-reusable.yml index b2afa7a6f60a..6a8813a0a343 100644 --- a/.github/workflows/ci-prover-reusable.yml +++ b/.github/workflows/ci-prover-reusable.yml @@ -60,6 +60,7 @@ jobs: - name: Init run: | ci_run zk + ci_run run_retried rustup show ci_run zk db setup - name: Prover unit tests From 6c49a50eb4374a06143e5bac130d0e0e74347597 Mon Sep 17 00:00:00 2001 From: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Date: Mon, 17 Jun 2024 19:04:49 +0300 Subject: [PATCH 05/43] fix(vm): Update `decommitted_code_hashes` in `prepare_to_decommit` (#2253) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Update `decommitted_code_hashes` in `prepare_to_decommit` ## Why ❔ Contract hashes that reached `prepare_to_decommit` should be returned by `get_used_contracts` ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- .../vm_latest/old_vm/oracles/decommitter.rs | 13 +++-- .../vm_latest/tests/get_used_contracts.rs | 51 ++++++++++++++++++- .../vm_latest/tests/tester/inner_state.rs | 2 +- .../vm_latest/tracers/circuits_tracer.rs | 27 +++++----- 4 files changed, 73 insertions(+), 20 deletions(-) diff --git a/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs b/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs index 85b18e203ce2..7c7dc6995d14 100644 --- a/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs +++ b/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs @@ -30,7 +30,7 @@ pub struct DecommitterOracle { /// Stores pages of memory where certain code hashes have already been decommitted. /// It is expected that they all are present in the DB. // `decommitted_code_hashes` history is necessary - pub decommitted_code_hashes: HistoryRecorder, HistoryEnabled>, + pub decommitted_code_hashes: HistoryRecorder>, HistoryEnabled>, /// Stores history of decommitment requests. decommitment_requests: HistoryRecorder, H>, } @@ -89,7 +89,7 @@ impl DecommitterOracle { pub fn get_decommitted_code_hashes_with_history( &self, - ) -> &HistoryRecorder, HistoryEnabled> { + ) -> &HistoryRecorder>, HistoryEnabled> { &self.decommitted_code_hashes } @@ -108,7 +108,7 @@ impl DecommitterOracle { .map(|(_, value)| value.len() * std::mem::size_of::()) .sum::(); let decommitted_code_hashes_size = - self.decommitted_code_hashes.inner().len() * std::mem::size_of::<(U256, u32)>(); + self.decommitted_code_hashes.inner().len() * std::mem::size_of::<(U256, Option)>(); known_bytecodes_size + decommitted_code_hashes_size } @@ -132,7 +132,7 @@ impl DecommitterOracle { ); let decommitted_code_hashes_size = self.decommitted_code_hashes.borrow_history(|h| h.len(), 0) - * std::mem::size_of::< as WithHistory>::HistoryRecord>(); + * std::mem::size_of::<> as WithHistory>::HistoryRecord>(); known_bytecodes_stack_size + known_bytecodes_heap_size + decommitted_code_hashes_size } @@ -172,6 +172,7 @@ impl DecommittmentProcess .inner() .get(&stored_hash) .copied() + .flatten() { partial_query.is_fresh = false; partial_query.memory_page = MemoryPage(memory_page); @@ -179,6 +180,8 @@ impl DecommittmentProcess Ok(partial_query) } else { partial_query.is_fresh = true; + self.decommitted_code_hashes + .insert(stored_hash, None, partial_query.timestamp); Ok(partial_query) } @@ -216,7 +219,7 @@ impl DecommittmentProcess rw_flag: true, }; self.decommitted_code_hashes - .insert(stored_hash, page_to_use.0, timestamp); + .insert(stored_hash, Some(page_to_use.0), timestamp); // Copy the bytecode (that is stored in 'values' Vec) into the memory page. if B { diff --git a/core/lib/multivm/src/versions/vm_latest/tests/get_used_contracts.rs b/core/lib/multivm/src/versions/vm_latest/tests/get_used_contracts.rs index 7bc08b6fb495..1798c700ea2d 100644 --- a/core/lib/multivm/src/versions/vm_latest/tests/get_used_contracts.rs +++ b/core/lib/multivm/src/versions/vm_latest/tests/get_used_contracts.rs @@ -1,6 +1,14 @@ -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + str::FromStr, +}; use itertools::Itertools; +use zk_evm_1_5_0::{ + abstractions::DecommittmentProcessor, + aux_structures::{DecommittmentQuery, MemoryPage, Timestamp}, + zkevm_opcode_defs::{VersionedHashHeader, VersionedHashNormalizedPreimage}, +}; use zksync_state::WriteStorage; use zksync_system_constants::CONTRACT_DEPLOYER_ADDRESS; use zksync_test_account::Account; @@ -91,6 +99,47 @@ fn test_get_used_contracts() { } } +#[test] +fn test_contract_is_used_right_after_prepare_to_decommit() { + let mut vm = VmTesterBuilder::new(HistoryDisabled) + .with_empty_in_memory_storage() + .with_execution_mode(TxExecutionMode::VerifyExecute) + .build(); + + assert!(vm.vm.get_used_contracts().is_empty()); + + let bytecode_hash = + U256::from_str("0x100067ff3124f394104ab03481f7923f0bc4029a2aa9d41cc1d848c81257185") + .unwrap(); + vm.vm + .state + .decommittment_processor + .populate(vec![(bytecode_hash, vec![])], Timestamp(0)); + + let header = hex::decode("0100067f").unwrap(); + let normalized_preimage = + hex::decode("f3124f394104ab03481f7923f0bc4029a2aa9d41cc1d848c81257185").unwrap(); + vm.vm + .state + .decommittment_processor + .prepare_to_decommit( + 0, + DecommittmentQuery { + header: VersionedHashHeader(header.try_into().unwrap()), + normalized_preimage: VersionedHashNormalizedPreimage( + normalized_preimage.try_into().unwrap(), + ), + timestamp: Timestamp(0), + memory_page: MemoryPage(0), + decommitted_length: 0, + is_fresh: false, + }, + ) + .unwrap(); + + assert_eq!(vm.vm.get_used_contracts(), vec![bytecode_hash]); +} + fn known_bytecodes_without_aa_code( vm: &Vm, ) -> HashMap> { diff --git a/core/lib/multivm/src/versions/vm_latest/tests/tester/inner_state.rs b/core/lib/multivm/src/versions/vm_latest/tests/tester/inner_state.rs index 102822351366..2a6fead8cf9c 100644 --- a/core/lib/multivm/src/versions/vm_latest/tests/tester/inner_state.rs +++ b/core/lib/multivm/src/versions/vm_latest/tests/tester/inner_state.rs @@ -43,7 +43,7 @@ pub(crate) struct DecommitterTestInnerState { /// so we just compare the modified keys. This is reasonable enough. pub(crate) modified_storage_keys: ModifiedKeysMap, pub(crate) known_bytecodes: HistoryRecorder>, H>, - pub(crate) decommitted_code_hashes: HistoryRecorder, HistoryEnabled>, + pub(crate) decommitted_code_hashes: HistoryRecorder>, HistoryEnabled>, } #[derive(Clone, PartialEq, Debug)] diff --git a/core/lib/multivm/src/versions/vm_latest/tracers/circuits_tracer.rs b/core/lib/multivm/src/versions/vm_latest/tracers/circuits_tracer.rs index 7c3012d03f11..4d5dc0b13273 100644 --- a/core/lib/multivm/src/versions/vm_latest/tracers/circuits_tracer.rs +++ b/core/lib/multivm/src/versions/vm_latest/tracers/circuits_tracer.rs @@ -177,20 +177,21 @@ impl CircuitsTracer { .decommitted_code_hashes .history(); for (_, history_event) in &history[last_decommitment_history_entry_checked..] { - // We assume that only insertions may happen during a single VM inspection. - assert!(history_event.value.is_none()); - let bytecode_len = state - .decommittment_processor - .known_bytecodes - .inner() - .get(&history_event.key) - .expect("Bytecode must be known at this point") - .len(); + // We update cycles once per bytecode when it is actually decommitted. + if history_event.value.is_some() { + let bytecode_len = state + .decommittment_processor + .known_bytecodes + .inner() + .get(&history_event.key) + .expect("Bytecode must be known at this point") + .len(); - // Each cycle of `CodeDecommitter` processes 2 words. - // If the number of words in bytecode is odd, then number of cycles must be rounded up. - let decommitter_cycles_used = (bytecode_len + 1) / 2; - self.statistics.code_decommitter_cycles += decommitter_cycles_used as u32; + // Each cycle of `CodeDecommitter` processes 2 words. + // If the number of words in bytecode is odd, then number of cycles must be rounded up. + let decommitter_cycles_used = (bytecode_len + 1) / 2; + self.statistics.code_decommitter_cycles += decommitter_cycles_used as u32; + } } self.last_decommitment_history_entry_checked = Some(history.len()); } From 63be1f3ba6b3ec8abf680ce2c84ba21227e5ade2 Mon Sep 17 00:00:00 2001 From: AntonD3 <74021421+AntonD3@users.noreply.github.com> Date: Mon, 17 Jun 2024 18:04:53 +0200 Subject: [PATCH 06/43] chore: move contract verifier logic to lib (#2240) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Move the contract verifier struct to the separate lib crate. ## Why ❔ This logic can be used as a library. At lest, for example, now it's needed for some compiler analysis tool. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- Cargo.lock | 25 +++++++++++--- Cargo.toml | 2 ++ core/bin/contract-verifier/Cargo.toml | 13 +------- core/bin/contract-verifier/src/main.rs | 9 +---- core/lib/contract_verifier/Cargo.toml | 33 +++++++++++++++++++ .../contract_verifier}/src/error.rs | 0 .../contract_verifier/src/lib.rs} | 7 +++- .../contract_verifier}/src/metrics.rs | 0 .../contract_verifier}/src/zksolc_utils.rs | 12 ------- .../contract_verifier}/src/zkvyper_utils.rs | 0 10 files changed, 63 insertions(+), 38 deletions(-) create mode 100644 core/lib/contract_verifier/Cargo.toml rename core/{bin/contract-verifier => lib/contract_verifier}/src/error.rs (100%) rename core/{bin/contract-verifier/src/verifier.rs => lib/contract_verifier/src/lib.rs} (99%) rename core/{bin/contract-verifier => lib/contract_verifier}/src/metrics.rs (100%) rename core/{bin/contract-verifier => lib/contract_verifier}/src/zksolc_utils.rs (97%) rename core/{bin/contract-verifier => lib/contract_verifier}/src/zkvyper_utils.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index cfe47a2a4b1e..a99150fe01c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8282,23 +8282,38 @@ name = "zksync_contract_verifier" version = "0.1.0" dependencies = [ "anyhow", - "chrono", "ctrlc", - "ethabi", "futures 0.3.28", + "prometheus_exporter", + "structopt", + "tokio", + "tracing", + "vlog", + "zksync_config", + "zksync_contract_verifier_lib", + "zksync_dal", + "zksync_env_config", + "zksync_queued_job_processor", + "zksync_utils", +] + +[[package]] +name = "zksync_contract_verifier_lib" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "ethabi", "hex", "lazy_static", - "prometheus_exporter", "regex", "serde", "serde_json", - "structopt", "tempfile", "thiserror", "tokio", "tracing", "vise", - "vlog", "zksync_config", "zksync_contracts", "zksync_dal", diff --git a/Cargo.toml b/Cargo.toml index de664288e150..5d9f6adf37ad 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ members = [ "core/lib/basic_types", "core/lib/config", "core/lib/constants", + "core/lib/contract_verifier", "core/lib/contracts", "core/lib/crypto", "core/lib/circuit_breaker", @@ -212,6 +213,7 @@ zksync = { path = "sdk/zksync-rs" } zksync_basic_types = { path = "core/lib/basic_types" } zksync_circuit_breaker = { path = "core/lib/circuit_breaker" } zksync_config = { path = "core/lib/config" } +zksync_contract_verifier_lib = { path = "core/lib/contract_verifier" } zksync_contracts = { path = "core/lib/contracts" } zksync_core_leftovers = { path = "core/lib/zksync_core_leftovers" } zksync_crypto = { path = "core/lib/crypto" } diff --git a/core/bin/contract-verifier/Cargo.toml b/core/bin/contract-verifier/Cargo.toml index 49e5469998c8..3e9832f995f9 100644 --- a/core/bin/contract-verifier/Cargo.toml +++ b/core/bin/contract-verifier/Cargo.toml @@ -11,11 +11,10 @@ categories.workspace = true publish = false [dependencies] -zksync_types.workspace = true zksync_dal.workspace = true zksync_env_config.workspace = true zksync_config.workspace = true -zksync_contracts.workspace = true +zksync_contract_verifier_lib.workspace = true zksync_queued_job_processor.workspace = true zksync_utils.workspace = true prometheus_exporter.workspace = true @@ -25,15 +24,5 @@ anyhow.workspace = true tokio = { workspace = true, features = ["full"] } futures.workspace = true ctrlc.workspace = true -thiserror.workspace = true -chrono.workspace = true -serde_json.workspace = true -ethabi.workspace = true -vise.workspace = true -hex.workspace = true -serde = { workspace = true, features = ["derive"] } structopt.workspace = true -lazy_static.workspace = true -tempfile.workspace = true -regex.workspace = true tracing.workspace = true diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 98b4a859d14d..5789422641c8 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -9,19 +9,12 @@ use zksync_config::{ configs::{ObservabilityConfig, PrometheusConfig}, ApiConfig, ContractVerifierConfig, }; +use zksync_contract_verifier_lib::ContractVerifier; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_env_config::FromEnv; use zksync_queued_job_processor::JobProcessor; use zksync_utils::{wait_for_tasks::ManagedTasks, workspace_dir_or_current_dir}; -use crate::verifier::ContractVerifier; - -pub mod error; -mod metrics; -pub mod verifier; -pub mod zksolc_utils; -pub mod zkvyper_utils; - async fn update_compiler_versions(connection_pool: &ConnectionPool) { let mut storage = connection_pool.connection().await.unwrap(); let mut transaction = storage.start_transaction().await.unwrap(); diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml new file mode 100644 index 000000000000..5b5ab4b5e429 --- /dev/null +++ b/core/lib/contract_verifier/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "zksync_contract_verifier_lib" +version = "0.1.0" +edition.workspace = true +authors.workspace = true +homepage.workspace = true +repository.workspace = true +license.workspace = true +keywords.workspace = true +categories.workspace = true + +[dependencies] +zksync_types.workspace = true +zksync_dal.workspace = true +zksync_env_config.workspace = true +zksync_config.workspace = true +zksync_contracts.workspace = true +zksync_queued_job_processor.workspace = true +zksync_utils.workspace = true + +anyhow.workspace = true +tokio = { workspace = true, features = ["full"] } +thiserror.workspace = true +chrono.workspace = true +serde_json.workspace = true +ethabi.workspace = true +vise.workspace = true +hex.workspace = true +serde = { workspace = true, features = ["derive"] } +lazy_static.workspace = true +tempfile.workspace = true +regex.workspace = true +tracing.workspace = true diff --git a/core/bin/contract-verifier/src/error.rs b/core/lib/contract_verifier/src/error.rs similarity index 100% rename from core/bin/contract-verifier/src/error.rs rename to core/lib/contract_verifier/src/error.rs diff --git a/core/bin/contract-verifier/src/verifier.rs b/core/lib/contract_verifier/src/lib.rs similarity index 99% rename from core/bin/contract-verifier/src/verifier.rs rename to core/lib/contract_verifier/src/lib.rs index 8d5ba9fccfe2..3ff4c2e18c77 100644 --- a/core/bin/contract-verifier/src/verifier.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -30,6 +30,11 @@ use crate::{ zkvyper_utils::{ZkVyper, ZkVyperInput}, }; +pub mod error; +mod metrics; +mod zksolc_utils; +mod zkvyper_utils; + lazy_static! { static ref DEPLOYER_CONTRACT: Contract = zksync_contracts::deployer_contract(); } @@ -274,7 +279,7 @@ impl ContractVerifier { Err(ContractVerifierError::MissingContract(contract_name)) } - async fn compile( + pub async fn compile( request: VerificationRequest, config: ContractVerifierConfig, ) -> Result { diff --git a/core/bin/contract-verifier/src/metrics.rs b/core/lib/contract_verifier/src/metrics.rs similarity index 100% rename from core/bin/contract-verifier/src/metrics.rs rename to core/lib/contract_verifier/src/metrics.rs diff --git a/core/bin/contract-verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs similarity index 97% rename from core/bin/contract-verifier/src/zksolc_utils.rs rename to core/lib/contract_verifier/src/zksolc_utils.rs index 791d5ee5b6cc..4952c1e21d0d 100644 --- a/core/bin/contract-verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -74,18 +74,6 @@ impl Default for Optimizer { } } -impl Optimizer { - /// - /// A shortcut constructor. - /// - pub fn new(enabled: bool) -> Self { - Self { - enabled, - mode: None, - } - } -} - pub struct ZkSolc { zksolc_path: PathBuf, solc_path: PathBuf, diff --git a/core/bin/contract-verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs similarity index 100% rename from core/bin/contract-verifier/src/zkvyper_utils.rs rename to core/lib/contract_verifier/src/zkvyper_utils.rs From f1d9f03ba32081d34a6a24e94b63fb494a33663e Mon Sep 17 00:00:00 2001 From: Danil Date: Mon, 17 Jun 2024 21:53:36 +0200 Subject: [PATCH 07/43] fix(zk_toolbox): Show balance (#2254) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Show balance of account if it's not enough for contract deployment ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. Signed-off-by: Danil --- zk_toolbox/crates/common/src/forge.rs | 8 ++++---- .../crates/zk_inception/src/forge_utils.rs | 16 +++++++++++----- .../crates/zk_inception/src/messages.rs | 19 ++++++++++++++++--- 3 files changed, 31 insertions(+), 12 deletions(-) diff --git a/zk_toolbox/crates/common/src/forge.rs b/zk_toolbox/crates/common/src/forge.rs index 3ae46a8034a8..565c7aa52d96 100644 --- a/zk_toolbox/crates/common/src/forge.rs +++ b/zk_toolbox/crates/common/src/forge.rs @@ -130,16 +130,16 @@ impl ForgeScript { }) } - pub async fn check_the_balance(&self, minimum_value: U256) -> anyhow::Result { + pub async fn get_the_balance(&self) -> anyhow::Result> { let Some(rpc_url) = self.rpc_url() else { - return Ok(true); + return Ok(None); }; let Some(private_key) = self.private_key() else { - return Ok(true); + return Ok(None); }; let client = create_ethers_client(private_key, rpc_url, None)?; let balance = client.get_balance(client.address(), None).await?; - Ok(balance > minimum_value) + Ok(Some(balance)) } } diff --git a/zk_toolbox/crates/zk_inception/src/forge_utils.rs b/zk_toolbox/crates/zk_inception/src/forge_utils.rs index 581d1ec892d4..cabc8ff7566b 100644 --- a/zk_toolbox/crates/zk_inception/src/forge_utils.rs +++ b/zk_toolbox/crates/zk_inception/src/forge_utils.rs @@ -22,11 +22,17 @@ pub async fn check_the_balance(forge: &ForgeScript) -> anyhow::Result<()> { return Ok(()); }; - while !forge - .check_the_balance(U256::from(MINIMUM_BALANCE_FOR_WALLET)) - .await? - { - if !common::PromptConfirm::new(msg_address_doesnt_have_enough_money_prompt(&address)).ask() + let expected_balance = U256::from(MINIMUM_BALANCE_FOR_WALLET); + while let Some(balance) = forge.get_the_balance().await? { + if balance >= expected_balance { + return Ok(()); + } + if !common::PromptConfirm::new(msg_address_doesnt_have_enough_money_prompt( + &address, + balance, + expected_balance, + )) + .ask() { break; } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 21f051470555..6d539d422bea 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -1,4 +1,7 @@ -use ethers::types::H160; +use ethers::{ + types::{H160, U256}, + utils::format_ether, +}; /// Common messages pub(super) const MSG_SELECTED_CONFIG: &str = "Selected config"; @@ -129,12 +132,15 @@ pub(super) const MSG_FAILED_TO_DROP_PROVER_DATABASE_ERR: &str = "Failed to drop pub(super) fn msg_server_db_url_prompt(chain_name: &str) -> String { format!("Please provide server database url for chain {chain_name}") } + pub(super) fn msg_prover_db_url_prompt(chain_name: &str) -> String { format!("Please provide prover database url for chain {chain_name}") } + pub(super) fn msg_prover_db_name_prompt(chain_name: &str) -> String { format!("Please provide prover database name for chain {chain_name}") } + pub(super) fn msg_server_db_name_prompt(chain_name: &str) -> String { format!("Please provide server database name for chain {chain_name}") } @@ -170,8 +176,15 @@ pub(super) const MSG_BUILDING_L1_CONTRACTS: &str = "Building L1 contracts..."; /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; -pub(super) fn msg_address_doesnt_have_enough_money_prompt(address: &H160) -> String { + +pub(super) fn msg_address_doesnt_have_enough_money_prompt( + address: &H160, + actual: U256, + expected: U256, +) -> String { + let actual = format_ether(actual); + let expected = format_ether(expected); format!( - "Address {address:?} doesn't have enough money to deploy contracts do you want to try again?" + "Address {address:?} doesn't have enough money to deploy contracts only {actual} ETH but expected: {expected} ETH do you want to try again?" ) } From 2f528ec8d49cb31ef714b409c703ae9f99cc5551 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Tue, 18 Jun 2024 11:20:52 +0200 Subject: [PATCH 08/43] feat(zk_toolbox): Use docker compose instead of docker-compose (#2195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Use `docker compose` instead of `docker-compose` ## Why ❔ The recommended command-line syntax is docker compose: https://docs.docker.com/compose/migrate/#docker-compose-vs-docker-compose --- zk_toolbox/crates/common/src/docker.rs | 4 ++-- zk_toolbox/crates/common/src/prerequisites.rs | 21 ++++++++++++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/zk_toolbox/crates/common/src/docker.rs b/zk_toolbox/crates/common/src/docker.rs index f52e3214fa23..db8a63e9f5d0 100644 --- a/zk_toolbox/crates/common/src/docker.rs +++ b/zk_toolbox/crates/common/src/docker.rs @@ -3,8 +3,8 @@ use xshell::{cmd, Shell}; use crate::cmd::Cmd; pub fn up(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { - Cmd::new(cmd!(shell, "docker-compose -f {docker_compose_file} up -d")).run() + Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} up -d")).run() } pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { - Cmd::new(cmd!(shell, "docker-compose -f {docker_compose_file} down")).run() + Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} down")).run() } diff --git a/zk_toolbox/crates/common/src/prerequisites.rs b/zk_toolbox/crates/common/src/prerequisites.rs index 237af5b40483..ae21ba68b3c1 100644 --- a/zk_toolbox/crates/common/src/prerequisites.rs +++ b/zk_toolbox/crates/common/src/prerequisites.rs @@ -2,7 +2,7 @@ use xshell::{cmd, Shell}; use crate::{cmd::Cmd, logger}; -const PREREQUISITES: [Prerequisite; 6] = [ +const PREREQUISITES: [Prerequisite; 5] = [ Prerequisite { name: "git", download_link: "https://git-scm.com/book/en/v2/Getting-Started-Installing-Git", @@ -11,10 +11,6 @@ const PREREQUISITES: [Prerequisite; 6] = [ name: "docker", download_link: "https://docs.docker.com/get-docker/", }, - Prerequisite { - name: "docker-compose", - download_link: "https://docs.docker.com/compose/install/", - }, Prerequisite { name: "forge", download_link: "https://book.getfoundry.sh/getting-started/installation", @@ -29,6 +25,11 @@ const PREREQUISITES: [Prerequisite; 6] = [ }, ]; +const DOCKER_COMPOSE_PREREQUISITE: Prerequisite = Prerequisite { + name: "docker compose", + download_link: "https://docs.docker.com/compose/install/", +}; + struct Prerequisite { name: &'static str, download_link: &'static str, @@ -43,6 +44,10 @@ pub fn check_prerequisites(shell: &Shell) { } } + if !check_docker_compose_prerequisite(shell) { + missing_prerequisites.push(&DOCKER_COMPOSE_PREREQUISITE); + } + if !missing_prerequisites.is_empty() { logger::error("Prerequisite check has failed"); logger::error_note( @@ -63,3 +68,9 @@ pub fn check_prerequisites(shell: &Shell) { fn check_prerequisite(shell: &Shell, name: &str) -> bool { Cmd::new(cmd!(shell, "which {name}")).run().is_ok() } + +fn check_docker_compose_prerequisite(shell: &Shell) -> bool { + Cmd::new(cmd!(shell, "docker compose version")) + .run() + .is_ok() +} From db8e71b55393b3d0e419886b62712b61305ac030 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti <56092489+ColoCarletti@users.noreply.github.com> Date: Tue, 18 Jun 2024 10:09:19 -0300 Subject: [PATCH 09/43] fix(prover_cli): Remove outdated fix for circuit id in node wg (#2248) This PR removes the fixes that were used to correct the circuit_id in the node witness generator. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- core/lib/basic_types/src/prover_dal.rs | 12 ------------ prover/prover_dal/src/fri_prover_dal.rs | 6 ++---- prover/prover_dal/src/fri_witness_generator_dal.rs | 11 +++++------ 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/core/lib/basic_types/src/prover_dal.rs b/core/lib/basic_types/src/prover_dal.rs index 1d741fac508c..5eb00dc63a4f 100644 --- a/core/lib/basic_types/src/prover_dal.rs +++ b/core/lib/basic_types/src/prover_dal.rs @@ -382,15 +382,3 @@ pub struct ProofCompressionJobInfo { pub time_taken: Option, pub picked_by: Option, } - -// This function corrects circuit IDs for the node witness generator. -// -// - Circuit IDs in the node witness generator are 2 higher than in other rounds. -// - The `EIP4844Repack` circuit (ID 255) is an exception and is set to 18. -pub fn correct_circuit_id(circuit_id: i16, aggregation_round: AggregationRound) -> u32 { - match (circuit_id, aggregation_round) { - (18, AggregationRound::NodeAggregation) => 255, - (circuit_id, AggregationRound::NodeAggregation) => (circuit_id as u32) - 2, - _ => circuit_id as u32, - } -} diff --git a/prover/prover_dal/src/fri_prover_dal.rs b/prover/prover_dal/src/fri_prover_dal.rs index f6c0379ee8a0..419cb635ac53 100644 --- a/prover/prover_dal/src/fri_prover_dal.rs +++ b/prover/prover_dal/src/fri_prover_dal.rs @@ -5,8 +5,7 @@ use zksync_basic_types::{ basic_fri_types::{AggregationRound, CircuitIdRoundTuple, JobIdentifiers}, protocol_version::{ProtocolSemanticVersion, ProtocolVersionId}, prover_dal::{ - correct_circuit_id, FriProverJobMetadata, JobCountStatistics, ProverJobFriInfo, - ProverJobStatus, StuckJobs, + FriProverJobMetadata, JobCountStatistics, ProverJobFriInfo, ProverJobStatus, StuckJobs, }, L1BatchNumber, }; @@ -659,8 +658,7 @@ impl FriProverDal<'_, '_> { .map(|row| ProverJobFriInfo { id: row.id as u32, l1_batch_number, - // It is necessary to correct the circuit IDs due to the discrepancy between different aggregation rounds. - circuit_id: correct_circuit_id(row.circuit_id, aggregation_round), + circuit_id: row.circuit_id as u32, circuit_blob_url: row.circuit_blob_url.clone(), aggregation_round, sequence_number: row.sequence_number as u32, diff --git a/prover/prover_dal/src/fri_witness_generator_dal.rs b/prover/prover_dal/src/fri_witness_generator_dal.rs index 14d47beed1a0..8db30e5a7f11 100644 --- a/prover/prover_dal/src/fri_witness_generator_dal.rs +++ b/prover/prover_dal/src/fri_witness_generator_dal.rs @@ -6,10 +6,10 @@ use zksync_basic_types::{ basic_fri_types::{AggregationRound, Eip4844Blobs}, protocol_version::{ProtocolSemanticVersion, ProtocolVersionId, VersionPatch}, prover_dal::{ - correct_circuit_id, BasicWitnessGeneratorJobInfo, JobCountStatistics, - LeafAggregationJobMetadata, LeafWitnessGeneratorJobInfo, NodeAggregationJobMetadata, - NodeWitnessGeneratorJobInfo, RecursionTipWitnessGeneratorJobInfo, - SchedulerWitnessGeneratorJobInfo, StuckJobs, WitnessJobStatus, + BasicWitnessGeneratorJobInfo, JobCountStatistics, LeafAggregationJobMetadata, + LeafWitnessGeneratorJobInfo, NodeAggregationJobMetadata, NodeWitnessGeneratorJobInfo, + RecursionTipWitnessGeneratorJobInfo, SchedulerWitnessGeneratorJobInfo, StuckJobs, + WitnessJobStatus, }, L1BatchNumber, }; @@ -1553,8 +1553,7 @@ impl FriWitnessGeneratorDal<'_, '_> { .map(|row| NodeWitnessGeneratorJobInfo { id: row.id as u32, l1_batch_number, - // It is necessary to correct the circuit IDs due to the discrepancy between different aggregation rounds. - circuit_id: correct_circuit_id(row.circuit_id, AggregationRound::NodeAggregation), + circuit_id: row.circuit_id as u32, depth: row.depth as u32, status: WitnessJobStatus::from_str(&row.status).unwrap(), attempts: row.attempts as u32, From 63efb2e530d8b1445bdd58537d6f0cdb5593cd75 Mon Sep 17 00:00:00 2001 From: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Date: Tue, 18 Jun 2024 16:13:12 +0300 Subject: [PATCH 10/43] feat(contract-verifier): Adjust contract verifier for zksolc 1.5.0 (#2255) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ - `system-mode`, `force-evmla` flag are not provided for zksolc post v1.5.0 compilations - `solc-path` is not provided for system yul post v1.5.0 compilations ## Why ❔ Changes in compiler interface ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- Cargo.lock | 1 + core/lib/contract_verifier/Cargo.toml | 1 + core/lib/contract_verifier/src/lib.rs | 6 +- .../lib/contract_verifier/src/zksolc_utils.rs | 92 ++++++++++++++++--- .../types/src/contract_verification_api.rs | 2 +- 5 files changed, 87 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a99150fe01c0..ccfb6715884e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8307,6 +8307,7 @@ dependencies = [ "hex", "lazy_static", "regex", + "semver", "serde", "serde_json", "tempfile", diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 5b5ab4b5e429..ea84024cba98 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -31,3 +31,4 @@ lazy_static.workspace = true tempfile.workspace = true regex.workspace = true tracing.workspace = true +semver.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 3ff4c2e18c77..224d4b292347 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -153,7 +153,11 @@ impl ContractVerifier { )); } - let zksolc = ZkSolc::new(zksolc_path, solc_path); + let zksolc = ZkSolc::new( + zksolc_path, + solc_path, + request.req.compiler_versions.zk_compiler_version(), + ); let output = time::timeout(config.compilation_timeout(), zksolc.async_compile(input)) .await diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index 4952c1e21d0d..08004632bcec 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, io::Write, path::PathBuf, process::Stdio}; +use semver::Version; use serde::{Deserialize, Serialize}; use crate::error::ContractVerifierError; @@ -77,13 +78,19 @@ impl Default for Optimizer { pub struct ZkSolc { zksolc_path: PathBuf, solc_path: PathBuf, + zksolc_version: String, } impl ZkSolc { - pub fn new(zksolc_path: impl Into, solc_path: impl Into) -> Self { + pub fn new( + zksolc_path: impl Into, + solc_path: impl Into, + zksolc_version: String, + ) -> Self { ZkSolc { zksolc_path: zksolc_path.into(), solc_path: solc_path.into(), + zksolc_version, } } @@ -93,26 +100,36 @@ impl ZkSolc { ) -> Result { use tokio::io::AsyncWriteExt; let mut command = tokio::process::Command::new(&self.zksolc_path); + command.stdout(Stdio::piped()).stderr(Stdio::piped()); + match &input { ZkSolcInput::StandardJson(input) => { - if input.settings.is_system { - command.arg("--system-mode"); - } - if input.settings.force_evmla { - command.arg("--force-evmla"); + if !self.is_post_1_5_0() { + if input.settings.is_system { + command.arg("--system-mode"); + } + if input.settings.force_evmla { + command.arg("--force-evmla"); + } } + + command.arg("--solc").arg(self.solc_path.to_str().unwrap()); } ZkSolcInput::YulSingleFile { is_system, .. } => { - if *is_system { - command.arg("--system-mode"); + if self.is_post_1_5_0() { + if *is_system { + command.arg("--enable-eravm-extensions"); + } else { + command.arg("--solc").arg(self.solc_path.to_str().unwrap()); + } + } else { + if *is_system { + command.arg("--system-mode"); + } + command.arg("--solc").arg(self.solc_path.to_str().unwrap()); } } } - command - .arg("--solc") - .arg(self.solc_path.to_str().unwrap()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); match input { ZkSolcInput::StandardJson(input) => { let mut child = command @@ -181,4 +198,53 @@ impl ZkSolc { } } } + + pub fn is_post_1_5_0(&self) -> bool { + // Special case + if &self.zksolc_version == "vm-1.5.0-a167aa3" { + false + } else if let Some(version) = self.zksolc_version.strip_prefix("v") { + if let Ok(semver) = Version::parse(version) { + let target = Version::new(1, 5, 0); + semver >= target + } else { + true + } + } else { + true + } + } +} + +#[cfg(test)] +mod tests { + use crate::zksolc_utils::ZkSolc; + + #[test] + fn check_is_post_1_5_0() { + // Special case. + let mut zksolc = ZkSolc::new(".", ".", "vm-1.5.0-a167aa3".to_string()); + assert!(!zksolc.is_post_1_5_0(), "vm-1.5.0-a167aa3"); + + zksolc.zksolc_version = "v1.5.0".to_string(); + assert!(zksolc.is_post_1_5_0(), "v1.5.0"); + + zksolc.zksolc_version = "v1.5.1".to_string(); + assert!(zksolc.is_post_1_5_0(), "v1.5.1"); + + zksolc.zksolc_version = "v1.10.1".to_string(); + assert!(zksolc.is_post_1_5_0(), "v1.10.1"); + + zksolc.zksolc_version = "v2.0.0".to_string(); + assert!(zksolc.is_post_1_5_0(), "v2.0.0"); + + zksolc.zksolc_version = "v1.4.15".to_string(); + assert!(!zksolc.is_post_1_5_0(), "v1.4.15"); + + zksolc.zksolc_version = "v1.3.21".to_string(); + assert!(!zksolc.is_post_1_5_0(), "v1.3.21"); + + zksolc.zksolc_version = "v0.5.1".to_string(); + assert!(!zksolc.is_post_1_5_0(), "v0.5.1"); + } } diff --git a/core/lib/types/src/contract_verification_api.rs b/core/lib/types/src/contract_verification_api.rs index 033bb9dc9f36..588de3cb675e 100644 --- a/core/lib/types/src/contract_verification_api.rs +++ b/core/lib/types/src/contract_verification_api.rs @@ -140,7 +140,7 @@ pub struct VerificationIncomingRequest { pub optimizer_mode: Option, #[serde(default)] pub constructor_arguments: Bytes, - #[serde(default)] + #[serde(default, alias = "enableEraVMExtensions")] pub is_system: bool, #[serde(default)] pub force_evmla: bool, From 26f2010ea2edd1cb79d80852c626051afc473c48 Mon Sep 17 00:00:00 2001 From: Danil Date: Tue, 18 Jun 2024 15:23:22 +0200 Subject: [PATCH 11/43] fix(zk_toolbox): Use the same l2 address for shared and erc20 bridge (#2260) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. Signed-off-by: Danil --- zk_toolbox/crates/zk_inception/src/config_manipulations.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs index 3c350fa8d894..a300a15e76c6 100644 --- a/zk_toolbox/crates/zk_inception/src/config_manipulations.rs +++ b/zk_toolbox/crates/zk_inception/src/config_manipulations.rs @@ -79,6 +79,8 @@ pub fn update_l2_shared_bridge( let mut contracts_config = ContractsConfig::read_with_base_path(shell, &config.configs)?; contracts_config.bridges.shared.l2_address = Some(initialize_bridges_output.l2_shared_bridge_proxy); + contracts_config.bridges.erc20.l2_address = + Some(initialize_bridges_output.l2_shared_bridge_proxy); contracts_config.save_with_base_path(shell, &config.configs)?; Ok(()) } From f05b0aefbb04ce715431bf039b8760e95f87dc93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Tue, 18 Jun 2024 17:40:40 +0200 Subject: [PATCH 12/43] feat(eth-sender): fix for missing eth_txs_history entries (#2236) This change introduces an invariant "If the transaction may have been sent, then the sent_at_block is set". This way we never remove eth_txs_history entries if the transaction may be included in block. The downside is that we can't distinguish between not sent transaction and one that has not been mined, but you can't have it both. The next step is to remove "send_unsent_transactions" step, but it can't be done in this PR as there might be transactions in db without sent_at_block set --------- Signed-off-by: tomg10 --- ...fc05eaa158a6f38a87187d7f2c2068a0112a.json} | 7 ++-- core/lib/dal/src/eth_sender_dal.rs | 9 ++++- core/node/eth_sender/src/eth_tx_manager.rs | 39 +++++++++---------- 3 files changed, 30 insertions(+), 25 deletions(-) rename core/lib/dal/.sqlx/{query-fe06e06c04466429bb85709e6fe8dd6c2ad2793c06071f4a067dcc31306adebc.json => query-45a968c6d667b13bbe9d895e7734fc05eaa158a6f38a87187d7f2c2068a0112a.json} (60%) diff --git a/core/lib/dal/.sqlx/query-fe06e06c04466429bb85709e6fe8dd6c2ad2793c06071f4a067dcc31306adebc.json b/core/lib/dal/.sqlx/query-45a968c6d667b13bbe9d895e7734fc05eaa158a6f38a87187d7f2c2068a0112a.json similarity index 60% rename from core/lib/dal/.sqlx/query-fe06e06c04466429bb85709e6fe8dd6c2ad2793c06071f4a067dcc31306adebc.json rename to core/lib/dal/.sqlx/query-45a968c6d667b13bbe9d895e7734fc05eaa158a6f38a87187d7f2c2068a0112a.json index 8f0065433012..36da129b5b77 100644 --- a/core/lib/dal/.sqlx/query-fe06e06c04466429bb85709e6fe8dd6c2ad2793c06071f4a067dcc31306adebc.json +++ b/core/lib/dal/.sqlx/query-45a968c6d667b13bbe9d895e7734fc05eaa158a6f38a87187d7f2c2068a0112a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO\n eth_txs_history (\n eth_tx_id,\n base_fee_per_gas,\n priority_fee_per_gas,\n tx_hash,\n signed_raw_tx,\n created_at,\n updated_at,\n blob_base_fee_per_gas\n )\n VALUES\n ($1, $2, $3, $4, $5, NOW(), NOW(), $6)\n ON CONFLICT (tx_hash) DO NOTHING\n RETURNING\n id\n ", + "query": "\n INSERT INTO\n eth_txs_history (\n eth_tx_id,\n base_fee_per_gas,\n priority_fee_per_gas,\n tx_hash,\n signed_raw_tx,\n created_at,\n updated_at,\n blob_base_fee_per_gas,\n sent_at_block,\n sent_at\n )\n VALUES\n ($1, $2, $3, $4, $5, NOW(), NOW(), $6, $7, NOW())\n ON CONFLICT (tx_hash) DO NOTHING\n RETURNING\n id\n ", "describe": { "columns": [ { @@ -16,12 +16,13 @@ "Int8", "Text", "Bytea", - "Int8" + "Int8", + "Int4" ] }, "nullable": [ false ] }, - "hash": "fe06e06c04466429bb85709e6fe8dd6c2ad2793c06071f4a067dcc31306adebc" + "hash": "45a968c6d667b13bbe9d895e7734fc05eaa158a6f38a87187d7f2c2068a0112a" } diff --git a/core/lib/dal/src/eth_sender_dal.rs b/core/lib/dal/src/eth_sender_dal.rs index ad1e910af12e..d32ed082131e 100644 --- a/core/lib/dal/src/eth_sender_dal.rs +++ b/core/lib/dal/src/eth_sender_dal.rs @@ -221,6 +221,7 @@ impl EthSenderDal<'_, '_> { Ok(eth_tx.into()) } + #[allow(clippy::too_many_arguments)] pub async fn insert_tx_history( &mut self, eth_tx_id: u32, @@ -229,6 +230,7 @@ impl EthSenderDal<'_, '_> { blob_base_fee_per_gas: Option, tx_hash: H256, raw_signed_tx: &[u8], + sent_at_block: u32, ) -> anyhow::Result> { let priority_fee_per_gas = i64::try_from(priority_fee_per_gas).context("Can't convert u64 to i64")?; @@ -247,10 +249,12 @@ impl EthSenderDal<'_, '_> { signed_raw_tx, created_at, updated_at, - blob_base_fee_per_gas + blob_base_fee_per_gas, + sent_at_block, + sent_at ) VALUES - ($1, $2, $3, $4, $5, NOW(), NOW(), $6) + ($1, $2, $3, $4, $5, NOW(), NOW(), $6, $7, NOW()) ON CONFLICT (tx_hash) DO NOTHING RETURNING id @@ -261,6 +265,7 @@ impl EthSenderDal<'_, '_> { tx_hash, raw_signed_tx, blob_base_fee_per_gas.map(|v| v as i64), + sent_at_block as i32 ) .fetch_optional(self.storage.conn()) .await? diff --git a/core/node/eth_sender/src/eth_tx_manager.rs b/core/node/eth_sender/src/eth_tx_manager.rs index ea07248aa813..f635d12bae13 100644 --- a/core/node/eth_sender/src/eth_tx_manager.rs +++ b/core/node/eth_sender/src/eth_tx_manager.rs @@ -190,12 +190,13 @@ impl EthTxManager { blob_base_fee_per_gas, signed_tx.hash, signed_tx.raw_tx.as_ref(), + current_block.0, ) .await .unwrap() { if let Err(error) = self - .send_raw_transaction(storage, tx_history_id, signed_tx.raw_tx, current_block) + .send_raw_transaction(storage, tx_history_id, signed_tx.raw_tx) .await { tracing::warn!( @@ -216,17 +217,9 @@ impl EthTxManager { storage: &mut Connection<'_, Core>, tx_history_id: u32, raw_tx: RawTransactionBytes, - current_block: L1BlockNumber, ) -> Result<(), EthSenderError> { match self.l1_interface.send_raw_tx(raw_tx).await { - Ok(_) => { - storage - .eth_sender_dal() - .set_sent_at_block(tx_history_id, current_block.0) - .await - .unwrap(); - Ok(()) - } + Ok(_) => Ok(()), Err(error) => { // In transient errors, server may have received the transaction // we don't want to loose record about it in case that happens @@ -401,16 +394,22 @@ impl EthTxManager { self.apply_tx_status(storage, ð_tx, tx_status, l1_block_numbers.finalized) .await; - } else if let Err(error) = self - .send_raw_transaction( - storage, - tx.id, - RawTransactionBytes::new_unchecked(tx.signed_raw_tx.clone()), - l1_block_numbers.latest, - ) - .await - { - tracing::warn!("Error sending transaction {tx:?}: {error}"); + } else { + storage + .eth_sender_dal() + .set_sent_at_block(tx.id, l1_block_numbers.latest.0) + .await + .unwrap(); + if let Err(error) = self + .send_raw_transaction( + storage, + tx.id, + RawTransactionBytes::new_unchecked(tx.signed_raw_tx.clone()), + ) + .await + { + tracing::warn!("Error sending transaction {tx:?}: {error}"); + } } } } From 2dac8463376b5ca7cb3aeefab83b9220f3b2466a Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 19 Jun 2024 09:52:44 +0400 Subject: [PATCH 13/43] fix(node_framework): Run gas adjuster task only if necessary (#2266) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Adds a check that would prevent gas adjuster task from actually running if there are no users of `GasAdjuster`. ## Why ❔ `GasAdjuster` is provided as a resource for anyone to use, but its support task uses RPC (which has usage limits), so it doesn't make sense to run it if nobody uses `GasAdjuster`. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- .../src/implementations/layers/l1_gas.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/core/node/node_framework/src/implementations/layers/l1_gas.rs b/core/node/node_framework/src/implementations/layers/l1_gas.rs index 8deafd4e2949..d465510eff5d 100644 --- a/core/node/node_framework/src/implementations/layers/l1_gas.rs +++ b/core/node/node_framework/src/implementations/layers/l1_gas.rs @@ -84,7 +84,17 @@ impl Task for GasAdjusterTask { "gas_adjuster".into() } - async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + // Gas adjuster layer is added to provide a resource for anyone to use, but it comes with + // a support task. If nobody has used the resource, we don't need to run the support task. + if Arc::strong_count(&self.gas_adjuster) == 1 { + tracing::info!( + "Gas adjuster is not used by any other task, not running the support task" + ); + stop_receiver.0.changed().await?; + return Ok(()); + } + self.gas_adjuster.run(stop_receiver.0).await } } From 40e0a956e86583a713d6aacdc61c625931f68e1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Wed, 19 Jun 2024 08:52:08 +0200 Subject: [PATCH 14/43] feat(zk_toolbox): Add prover generate-sk command (#2222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- .../crates/zk_inception/src/commands/mod.rs | 1 + .../src/commands/prover/generate_sk.rs | 27 +++++++++++++++++++ .../zk_inception/src/commands/prover/mod.rs | 16 +++++++++++ .../zk_inception/src/commands/prover/utils.rs | 10 +++++++ zk_toolbox/crates/zk_inception/src/main.rs | 8 +++++- .../crates/zk_inception/src/messages.rs | 4 +++ 6 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index 8ed7a82b8334..ccdf5b082caa 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -2,4 +2,5 @@ pub mod args; pub mod chain; pub mod containers; pub mod ecosystem; +pub mod prover; pub mod server; diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs new file mode 100644 index 000000000000..a14dd6fb87e5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/generate_sk.rs @@ -0,0 +1,27 @@ +use anyhow::Ok; +use common::{cmd::Cmd, logger, spinner::Spinner}; +use config::EcosystemConfig; +use xshell::{cmd, Shell}; + +use super::utils::get_link_to_prover; +use crate::messages::{MSG_GENERATING_SK_SPINNER, MSG_SK_GENERATED}; + +pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let link_to_prover = get_link_to_prover(&ecosystem_config); + shell.change_dir(&link_to_prover); + + let spinner = Spinner::new(MSG_GENERATING_SK_SPINNER); + let mut cmd = Cmd::new(cmd!( + shell, + "cargo run --features gpu --release --bin key_generator -- + generate-sk all --recompute-if-missing + --setup-path=vk_setup_data_generator_server_fri/data + --path={link_to_prover}/vk_setup_data_generator_server_fri/data" + )); + cmd.run()?; + spinner.finish(); + logger::outro(MSG_SK_GENERATED); + + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs new file mode 100644 index 000000000000..c617b915a52c --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/mod.rs @@ -0,0 +1,16 @@ +use clap::Subcommand; +use xshell::Shell; +mod generate_sk; +mod utils; + +#[derive(Subcommand, Debug)] +pub enum ProverCommands { + /// Initialize prover + GenerateSK, +} + +pub(crate) async fn run(shell: &Shell, args: ProverCommands) -> anyhow::Result<()> { + match args { + ProverCommands::GenerateSK => generate_sk::run(shell).await, + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs b/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs new file mode 100644 index 000000000000..4dae70863dc9 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/prover/utils.rs @@ -0,0 +1,10 @@ +use std::path::PathBuf; + +use config::EcosystemConfig; + +pub(crate) fn get_link_to_prover(config: &EcosystemConfig) -> PathBuf { + let link_to_code = config.link_to_code.clone(); + let mut link_to_prover = link_to_code.into_os_string(); + link_to_prover.push("/prover"); + link_to_prover.into() +} diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index b0e8e8f4fd69..dff9e479e01f 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -7,7 +7,9 @@ use common::{ use config::EcosystemConfig; use xshell::Shell; -use crate::commands::{args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands}; +use crate::commands::{ + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, prover::ProverCommands, +}; pub mod accept_ownership; mod commands; @@ -35,6 +37,9 @@ pub enum InceptionSubcommands { /// Chain related commands #[command(subcommand)] Chain(ChainCommands), + /// Prover related commands + #[command(subcommand)] + Prover(ProverCommands), /// Run server Server(RunServerArgs), /// Run containers for local development @@ -101,6 +106,7 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res match inception_args.command { InceptionSubcommands::Ecosystem(args) => commands::ecosystem::run(shell, args).await?, InceptionSubcommands::Chain(args) => commands::chain::run(shell, args).await?, + InceptionSubcommands::Prover(args) => commands::prover::run(shell, args).await?, InceptionSubcommands::Server(args) => commands::server::run(shell, args)?, InceptionSubcommands::Containers => commands::containers::run(shell)?, } diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 6d539d422bea..1b3c05258753 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -188,3 +188,7 @@ pub(super) fn msg_address_doesnt_have_enough_money_prompt( "Address {address:?} doesn't have enough money to deploy contracts only {actual} ETH but expected: {expected} ETH do you want to try again?" ) } + +/// Prover related messages +pub(super) const MSG_GENERATING_SK_SPINNER: &str = "Generating setup keys..."; +pub(super) const MSG_SK_GENERATED: &str = "Setup keys generated successfully"; From 496e6c1aa5c10ec263102bdbf5b2cc18a87808b7 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 19 Jun 2024 09:45:28 +0200 Subject: [PATCH 15/43] fix(nix): make devShells.default `pure` again (#2269) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ this removes the need to call `nix develop` with the `--impure` flag ## Why ❔ This removes an inconvenience. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. Signed-off-by: Harald Hoyer --- docs/guides/setup-dev.md | 3 +-- flake.nix | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/guides/setup-dev.md b/docs/guides/setup-dev.md index 7b2879ff04a4..b8db0c1575c7 100644 --- a/docs/guides/setup-dev.md +++ b/docs/guides/setup-dev.md @@ -258,8 +258,7 @@ Install `nix`. Enable the nix command and flakes. Install docker, rustup and use rust to install SQLx CLI like described above. If you are on NixOS, you also need to enable nix-ld. -Go to the zksync folder and run `nix develop --impure`. After it finishes, you are in a shell that has all the -dependencies. +Go to the zksync folder and run `nix develop`. After it finishes, you are in a shell that has all the dependencies. ## Foundry diff --git a/flake.nix b/flake.nix index 4a056129687d..018aebb15da9 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,7 @@ # $ nix build .#zksync_server.block_reverter # # To enter the development shell, run: -# $ nix develop --impure +# $ nix develop # # To vendor the dependencies manually, run: # $ nix shell .#cargo-vendor -c cargo vendor --no-merge-sources @@ -212,7 +212,7 @@ export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER="clang" if [ "x$NIX_LD" = "x" ]; then - export NIX_LD="$ZK_NIX_LD" + export NIX_LD="$(<${clangStdenv.cc}/nix-support/dynamic-linker)" fi if [ "x$NIX_LD_LIBRARY_PATH" = "x" ]; then export NIX_LD_LIBRARY_PATH="$ZK_NIX_LD_LIBRARY_PATH" @@ -222,7 +222,6 @@ ''; ZK_NIX_LD_LIBRARY_PATH = lib.makeLibraryPath [ ]; - ZK_NIX_LD = builtins.readFile "${clangStdenv.cc}/nix-support/dynamic-linker"; }; }; }); From 0d51cd6f3e65eef1bda981fe96f3026d8e12156d Mon Sep 17 00:00:00 2001 From: Marcin M <128217157+mm-zk@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:00:06 +0200 Subject: [PATCH 16/43] feat: Expose fair_pubdata_price for blocks and batches (#2244) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ * Exposing fair_pubdata_price in our block & batches API ## Why ❔ * This is crucial information for projects that need to replay our transactions (for example era_test_node). * Without this, they cannot compute the correct gas costs. --- core/bin/external_node/src/tests.rs | 1 + ...3db7a71aca15698bafba051a8d9a91a4dbc76.json | 112 ++++++++++++++++++ ...c5437752a4cf3ac92ec09b334089a8dc5b4ca.json | 106 ----------------- ...8222bd9fbe8ce82d8963f7da03fe6fcf9225.json} | 16 ++- core/lib/dal/src/blocks_web3_dal.rs | 5 +- core/lib/dal/src/models/storage_block.rs | 5 + core/lib/snapshots_applier/src/tests/utils.rs | 1 + core/lib/types/src/api/mod.rs | 2 + .../src/batch_status_updater/tests.rs | 1 + .../src/tree_data_fetcher/provider/tests.rs | 1 + 10 files changed, 138 insertions(+), 112 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-1074d0a2e4a4afb9a92f3822e133db7a71aca15698bafba051a8d9a91a4dbc76.json delete mode 100644 core/lib/dal/.sqlx/query-44490ad52b8dbcd978a96677ffac5437752a4cf3ac92ec09b334089a8dc5b4ca.json rename core/lib/dal/.sqlx/{query-6874b501c82e6062ab22622095070d67840b2484ea3a03ac49eb3d50ea153163.json => query-ef70506e90e8add3b95940a7333f8222bd9fbe8ce82d8963f7da03fe6fcf9225.json} (72%) diff --git a/core/bin/external_node/src/tests.rs b/core/bin/external_node/src/tests.rs index 6611ce145c4c..c78c5329386e 100644 --- a/core/bin/external_node/src/tests.rs +++ b/core/bin/external_node/src/tests.rs @@ -34,6 +34,7 @@ fn block_details_base(hash: H256) -> api::BlockDetailsBase { executed_at: None, l1_gas_price: 0, l2_fair_gas_price: 0, + fair_pubdata_price: None, base_system_contracts_hashes: Default::default(), } } diff --git a/core/lib/dal/.sqlx/query-1074d0a2e4a4afb9a92f3822e133db7a71aca15698bafba051a8d9a91a4dbc76.json b/core/lib/dal/.sqlx/query-1074d0a2e4a4afb9a92f3822e133db7a71aca15698bafba051a8d9a91a4dbc76.json new file mode 100644 index 000000000000..13e4cdb9431d --- /dev/null +++ b/core/lib/dal/.sqlx/query-1074d0a2e4a4afb9a92f3822e133db7a71aca15698bafba051a8d9a91a4dbc76.json @@ -0,0 +1,112 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH\n mb AS (\n SELECT\n l1_gas_price,\n l2_fair_gas_price,\n fair_pubdata_price\n FROM\n miniblocks\n WHERE\n l1_batch_number = $1\n LIMIT\n 1\n )\n SELECT\n l1_batches.number,\n l1_batches.timestamp,\n l1_batches.l1_tx_count,\n l1_batches.l2_tx_count,\n l1_batches.hash AS \"root_hash?\",\n commit_tx.tx_hash AS \"commit_tx_hash?\",\n commit_tx.confirmed_at AS \"committed_at?\",\n prove_tx.tx_hash AS \"prove_tx_hash?\",\n prove_tx.confirmed_at AS \"proven_at?\",\n execute_tx.tx_hash AS \"execute_tx_hash?\",\n execute_tx.confirmed_at AS \"executed_at?\",\n mb.l1_gas_price,\n mb.l2_fair_gas_price,\n mb.fair_pubdata_price,\n l1_batches.bootloader_code_hash,\n l1_batches.default_aa_code_hash\n FROM\n l1_batches\n INNER JOIN mb ON TRUE\n LEFT JOIN eth_txs_history AS commit_tx ON (\n l1_batches.eth_commit_tx_id = commit_tx.eth_tx_id\n AND commit_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS prove_tx ON (\n l1_batches.eth_prove_tx_id = prove_tx.eth_tx_id\n AND prove_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS execute_tx ON (\n l1_batches.eth_execute_tx_id = execute_tx.eth_tx_id\n AND execute_tx.confirmed_at IS NOT NULL\n )\n WHERE\n l1_batches.number = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "number", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "timestamp", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "l1_tx_count", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "l2_tx_count", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "root_hash?", + "type_info": "Bytea" + }, + { + "ordinal": 5, + "name": "commit_tx_hash?", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "committed_at?", + "type_info": "Timestamp" + }, + { + "ordinal": 7, + "name": "prove_tx_hash?", + "type_info": "Text" + }, + { + "ordinal": 8, + "name": "proven_at?", + "type_info": "Timestamp" + }, + { + "ordinal": 9, + "name": "execute_tx_hash?", + "type_info": "Text" + }, + { + "ordinal": 10, + "name": "executed_at?", + "type_info": "Timestamp" + }, + { + "ordinal": 11, + "name": "l1_gas_price", + "type_info": "Int8" + }, + { + "ordinal": 12, + "name": "l2_fair_gas_price", + "type_info": "Int8" + }, + { + "ordinal": 13, + "name": "fair_pubdata_price", + "type_info": "Int8" + }, + { + "ordinal": 14, + "name": "bootloader_code_hash", + "type_info": "Bytea" + }, + { + "ordinal": 15, + "name": "default_aa_code_hash", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false, + true, + false, + true, + false, + true, + false, + false, + true, + true, + true + ] + }, + "hash": "1074d0a2e4a4afb9a92f3822e133db7a71aca15698bafba051a8d9a91a4dbc76" +} diff --git a/core/lib/dal/.sqlx/query-44490ad52b8dbcd978a96677ffac5437752a4cf3ac92ec09b334089a8dc5b4ca.json b/core/lib/dal/.sqlx/query-44490ad52b8dbcd978a96677ffac5437752a4cf3ac92ec09b334089a8dc5b4ca.json deleted file mode 100644 index cb2d1b149ecf..000000000000 --- a/core/lib/dal/.sqlx/query-44490ad52b8dbcd978a96677ffac5437752a4cf3ac92ec09b334089a8dc5b4ca.json +++ /dev/null @@ -1,106 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH\n mb AS (\n SELECT\n l1_gas_price,\n l2_fair_gas_price\n FROM\n miniblocks\n WHERE\n l1_batch_number = $1\n LIMIT\n 1\n )\n SELECT\n l1_batches.number,\n l1_batches.timestamp,\n l1_batches.l1_tx_count,\n l1_batches.l2_tx_count,\n l1_batches.hash AS \"root_hash?\",\n commit_tx.tx_hash AS \"commit_tx_hash?\",\n commit_tx.confirmed_at AS \"committed_at?\",\n prove_tx.tx_hash AS \"prove_tx_hash?\",\n prove_tx.confirmed_at AS \"proven_at?\",\n execute_tx.tx_hash AS \"execute_tx_hash?\",\n execute_tx.confirmed_at AS \"executed_at?\",\n mb.l1_gas_price,\n mb.l2_fair_gas_price,\n l1_batches.bootloader_code_hash,\n l1_batches.default_aa_code_hash\n FROM\n l1_batches\n INNER JOIN mb ON TRUE\n LEFT JOIN eth_txs_history AS commit_tx ON (\n l1_batches.eth_commit_tx_id = commit_tx.eth_tx_id\n AND commit_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS prove_tx ON (\n l1_batches.eth_prove_tx_id = prove_tx.eth_tx_id\n AND prove_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS execute_tx ON (\n l1_batches.eth_execute_tx_id = execute_tx.eth_tx_id\n AND execute_tx.confirmed_at IS NOT NULL\n )\n WHERE\n l1_batches.number = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "number", - "type_info": "Int8" - }, - { - "ordinal": 1, - "name": "timestamp", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "l1_tx_count", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "l2_tx_count", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "root_hash?", - "type_info": "Bytea" - }, - { - "ordinal": 5, - "name": "commit_tx_hash?", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "committed_at?", - "type_info": "Timestamp" - }, - { - "ordinal": 7, - "name": "prove_tx_hash?", - "type_info": "Text" - }, - { - "ordinal": 8, - "name": "proven_at?", - "type_info": "Timestamp" - }, - { - "ordinal": 9, - "name": "execute_tx_hash?", - "type_info": "Text" - }, - { - "ordinal": 10, - "name": "executed_at?", - "type_info": "Timestamp" - }, - { - "ordinal": 11, - "name": "l1_gas_price", - "type_info": "Int8" - }, - { - "ordinal": 12, - "name": "l2_fair_gas_price", - "type_info": "Int8" - }, - { - "ordinal": 13, - "name": "bootloader_code_hash", - "type_info": "Bytea" - }, - { - "ordinal": 14, - "name": "default_aa_code_hash", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Int8" - ] - }, - "nullable": [ - false, - false, - false, - false, - true, - false, - true, - false, - true, - false, - true, - false, - false, - true, - true - ] - }, - "hash": "44490ad52b8dbcd978a96677ffac5437752a4cf3ac92ec09b334089a8dc5b4ca" -} diff --git a/core/lib/dal/.sqlx/query-6874b501c82e6062ab22622095070d67840b2484ea3a03ac49eb3d50ea153163.json b/core/lib/dal/.sqlx/query-ef70506e90e8add3b95940a7333f8222bd9fbe8ce82d8963f7da03fe6fcf9225.json similarity index 72% rename from core/lib/dal/.sqlx/query-6874b501c82e6062ab22622095070d67840b2484ea3a03ac49eb3d50ea153163.json rename to core/lib/dal/.sqlx/query-ef70506e90e8add3b95940a7333f8222bd9fbe8ce82d8963f7da03fe6fcf9225.json index 5ccda40f56fc..cf102b828aa8 100644 --- a/core/lib/dal/.sqlx/query-6874b501c82e6062ab22622095070d67840b2484ea3a03ac49eb3d50ea153163.json +++ b/core/lib/dal/.sqlx/query-ef70506e90e8add3b95940a7333f8222bd9fbe8ce82d8963f7da03fe6fcf9225.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n miniblocks.number,\n COALESCE(\n miniblocks.l1_batch_number,\n (\n SELECT\n (MAX(number) + 1)\n FROM\n l1_batches\n )\n ) AS \"l1_batch_number!\",\n miniblocks.timestamp,\n miniblocks.l1_tx_count,\n miniblocks.l2_tx_count,\n miniblocks.hash AS \"root_hash?\",\n commit_tx.tx_hash AS \"commit_tx_hash?\",\n commit_tx.confirmed_at AS \"committed_at?\",\n prove_tx.tx_hash AS \"prove_tx_hash?\",\n prove_tx.confirmed_at AS \"proven_at?\",\n execute_tx.tx_hash AS \"execute_tx_hash?\",\n execute_tx.confirmed_at AS \"executed_at?\",\n miniblocks.l1_gas_price,\n miniblocks.l2_fair_gas_price,\n miniblocks.bootloader_code_hash,\n miniblocks.default_aa_code_hash,\n miniblocks.protocol_version,\n miniblocks.fee_account_address\n FROM\n miniblocks\n LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number\n LEFT JOIN eth_txs_history AS commit_tx ON (\n l1_batches.eth_commit_tx_id = commit_tx.eth_tx_id\n AND commit_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS prove_tx ON (\n l1_batches.eth_prove_tx_id = prove_tx.eth_tx_id\n AND prove_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS execute_tx ON (\n l1_batches.eth_execute_tx_id = execute_tx.eth_tx_id\n AND execute_tx.confirmed_at IS NOT NULL\n )\n WHERE\n miniblocks.number = $1\n ", + "query": "\n SELECT\n miniblocks.number,\n COALESCE(\n miniblocks.l1_batch_number,\n (\n SELECT\n (MAX(number) + 1)\n FROM\n l1_batches\n )\n ) AS \"l1_batch_number!\",\n miniblocks.timestamp,\n miniblocks.l1_tx_count,\n miniblocks.l2_tx_count,\n miniblocks.hash AS \"root_hash?\",\n commit_tx.tx_hash AS \"commit_tx_hash?\",\n commit_tx.confirmed_at AS \"committed_at?\",\n prove_tx.tx_hash AS \"prove_tx_hash?\",\n prove_tx.confirmed_at AS \"proven_at?\",\n execute_tx.tx_hash AS \"execute_tx_hash?\",\n execute_tx.confirmed_at AS \"executed_at?\",\n miniblocks.l1_gas_price,\n miniblocks.l2_fair_gas_price,\n miniblocks.fair_pubdata_price,\n miniblocks.bootloader_code_hash,\n miniblocks.default_aa_code_hash,\n miniblocks.protocol_version,\n miniblocks.fee_account_address\n FROM\n miniblocks\n LEFT JOIN l1_batches ON miniblocks.l1_batch_number = l1_batches.number\n LEFT JOIN eth_txs_history AS commit_tx ON (\n l1_batches.eth_commit_tx_id = commit_tx.eth_tx_id\n AND commit_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS prove_tx ON (\n l1_batches.eth_prove_tx_id = prove_tx.eth_tx_id\n AND prove_tx.confirmed_at IS NOT NULL\n )\n LEFT JOIN eth_txs_history AS execute_tx ON (\n l1_batches.eth_execute_tx_id = execute_tx.eth_tx_id\n AND execute_tx.confirmed_at IS NOT NULL\n )\n WHERE\n miniblocks.number = $1\n ", "describe": { "columns": [ { @@ -75,21 +75,26 @@ }, { "ordinal": 14, + "name": "fair_pubdata_price", + "type_info": "Int8" + }, + { + "ordinal": 15, "name": "bootloader_code_hash", "type_info": "Bytea" }, { - "ordinal": 15, + "ordinal": 16, "name": "default_aa_code_hash", "type_info": "Bytea" }, { - "ordinal": 16, + "ordinal": 17, "name": "protocol_version", "type_info": "Int4" }, { - "ordinal": 17, + "ordinal": 18, "name": "fee_account_address", "type_info": "Bytea" } @@ -117,8 +122,9 @@ true, true, true, + true, false ] }, - "hash": "6874b501c82e6062ab22622095070d67840b2484ea3a03ac49eb3d50ea153163" + "hash": "ef70506e90e8add3b95940a7333f8222bd9fbe8ce82d8963f7da03fe6fcf9225" } diff --git a/core/lib/dal/src/blocks_web3_dal.rs b/core/lib/dal/src/blocks_web3_dal.rs index f7b88f94a673..1c7f912728cc 100644 --- a/core/lib/dal/src/blocks_web3_dal.rs +++ b/core/lib/dal/src/blocks_web3_dal.rs @@ -629,6 +629,7 @@ impl BlocksWeb3Dal<'_, '_> { execute_tx.confirmed_at AS "executed_at?", miniblocks.l1_gas_price, miniblocks.l2_fair_gas_price, + miniblocks.fair_pubdata_price, miniblocks.bootloader_code_hash, miniblocks.default_aa_code_hash, miniblocks.protocol_version, @@ -673,7 +674,8 @@ impl BlocksWeb3Dal<'_, '_> { mb AS ( SELECT l1_gas_price, - l2_fair_gas_price + l2_fair_gas_price, + fair_pubdata_price FROM miniblocks WHERE @@ -695,6 +697,7 @@ impl BlocksWeb3Dal<'_, '_> { execute_tx.confirmed_at AS "executed_at?", mb.l1_gas_price, mb.l2_fair_gas_price, + mb.fair_pubdata_price, l1_batches.bootloader_code_hash, l1_batches.default_aa_code_hash FROM diff --git a/core/lib/dal/src/models/storage_block.rs b/core/lib/dal/src/models/storage_block.rs index de6d1d9f06cd..95780e667784 100644 --- a/core/lib/dal/src/models/storage_block.rs +++ b/core/lib/dal/src/models/storage_block.rs @@ -269,6 +269,8 @@ pub(crate) struct StorageBlockDetails { pub l1_gas_price: i64, // L2 gas price assumed in the corresponding batch pub l2_fair_gas_price: i64, + // Cost of publishing 1 byte (in wei). + pub fair_pubdata_price: Option, pub bootloader_code_hash: Option>, pub default_aa_code_hash: Option>, pub fee_account_address: Vec, @@ -312,6 +314,7 @@ impl From for api::BlockDetails { .map(|executed_at| DateTime::::from_naive_utc_and_offset(executed_at, Utc)), l1_gas_price: details.l1_gas_price as u64, l2_fair_gas_price: details.l2_fair_gas_price as u64, + fair_pubdata_price: details.fair_pubdata_price.map(|x| x as u64), base_system_contracts_hashes: convert_base_system_contracts_hashes( details.bootloader_code_hash, details.default_aa_code_hash, @@ -344,6 +347,7 @@ pub(crate) struct StorageL1BatchDetails { pub executed_at: Option, pub l1_gas_price: i64, pub l2_fair_gas_price: i64, + pub fair_pubdata_price: Option, pub bootloader_code_hash: Option>, pub default_aa_code_hash: Option>, } @@ -385,6 +389,7 @@ impl From for api::L1BatchDetails { .map(|executed_at| DateTime::::from_naive_utc_and_offset(executed_at, Utc)), l1_gas_price: details.l1_gas_price as u64, l2_fair_gas_price: details.l2_fair_gas_price as u64, + fair_pubdata_price: details.fair_pubdata_price.map(|x| x as u64), base_system_contracts_hashes: convert_base_system_contracts_hashes( details.bootloader_code_hash, details.default_aa_code_hash, diff --git a/core/lib/snapshots_applier/src/tests/utils.rs b/core/lib/snapshots_applier/src/tests/utils.rs index b48277a88e52..e683e0cae00f 100644 --- a/core/lib/snapshots_applier/src/tests/utils.rs +++ b/core/lib/snapshots_applier/src/tests/utils.rs @@ -156,6 +156,7 @@ fn block_details_base(hash: H256) -> api::BlockDetailsBase { executed_at: None, l1_gas_price: 0, l2_fair_gas_price: 0, + fair_pubdata_price: None, base_system_contracts_hashes: Default::default(), } } diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 5c0bfe2d848b..6e22e17de673 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -761,6 +761,8 @@ pub struct BlockDetailsBase { pub executed_at: Option>, pub l1_gas_price: u64, pub l2_fair_gas_price: u64, + // Cost of publishing one byte (in wei). + pub fair_pubdata_price: Option, pub base_system_contracts_hashes: BaseSystemContractsHashes, } diff --git a/core/node/node_sync/src/batch_status_updater/tests.rs b/core/node/node_sync/src/batch_status_updater/tests.rs index e1386f985a09..28b89f86f6a7 100644 --- a/core/node/node_sync/src/batch_status_updater/tests.rs +++ b/core/node/node_sync/src/batch_status_updater/tests.rs @@ -158,6 +158,7 @@ fn mock_block_details(number: u32, stage: L1BatchStage) -> api::BlockDetails { .then(|| Utc.timestamp_opt(300, 0).unwrap()), l1_gas_price: 1, l2_fair_gas_price: 2, + fair_pubdata_price: None, base_system_contracts_hashes: BaseSystemContractsHashes::default(), }, operator_address: Address::zero(), diff --git a/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs b/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs index bb252e09caad..291bc71c897d 100644 --- a/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs +++ b/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs @@ -39,6 +39,7 @@ fn mock_block_details_base(number: u32, hash: Option) -> api::BlockDetails executed_at: None, l1_gas_price: 10, l2_fair_gas_price: 100, + fair_pubdata_price: None, base_system_contracts_hashes: Default::default(), } } From 9cc757a7af6897eecfad51a9b27afab15fd9d945 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 19 Jun 2024 11:10:13 +0300 Subject: [PATCH 17/43] =?UTF-8?q?refactor(en):=20Fetch=20old=20L1=20batch?= =?UTF-8?q?=20hashes=20from=20L1=20=E2=80=93=20improve=20metrics=20/=20log?= =?UTF-8?q?ging=20(#2242)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Improves logging and metrics for the tree data fetcher. ## Why ❔ Currently, some errors don't have their context logged, and some metrics are defined suboptimally. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- .../src/tree_data_fetcher/metrics.rs | 12 ++- .../node_sync/src/tree_data_fetcher/mod.rs | 18 ++-- .../src/tree_data_fetcher/provider/mod.rs | 92 +++++++++---------- .../src/tree_data_fetcher/provider/tests.rs | 27 +++--- .../node_sync/src/tree_data_fetcher/tests.rs | 18 +--- 5 files changed, 79 insertions(+), 88 deletions(-) diff --git a/core/node/node_sync/src/tree_data_fetcher/metrics.rs b/core/node/node_sync/src/tree_data_fetcher/metrics.rs index 37c81cd2d40a..aad5f090e1fc 100644 --- a/core/node/node_sync/src/tree_data_fetcher/metrics.rs +++ b/core/node/node_sync/src/tree_data_fetcher/metrics.rs @@ -7,7 +7,7 @@ use vise::{ Info, Metrics, Unit, }; -use super::{provider::TreeDataProviderSource, StepOutcome, TreeDataFetcher, TreeDataFetcherError}; +use super::{StepOutcome, TreeDataFetcher, TreeDataFetcherError}; #[derive(Debug, EncodeLabelSet)] struct TreeDataFetcherInfo { @@ -30,6 +30,9 @@ impl From<&TreeDataFetcher> for TreeDataFetcherInfo { #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)] #[metrics(label = "stage", rename_all = "snake_case")] pub(super) enum ProcessingStage { + FetchL1CommitEvent, + FetchBatchDetailsRpc, + /// Total latency for all clients. Fetch, Persistence, } @@ -44,6 +47,13 @@ pub(super) enum StepOutcomeLabel { TransientError, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)] +#[metrics(label = "source", rename_all = "snake_case")] +pub(super) enum TreeDataProviderSource { + L1CommitEvent, + BatchDetailsRpc, +} + const BLOCK_DIFF_BUCKETS: Buckets = Buckets::values(&[ 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1_000.0, 2_000.0, 5_000.0, 10_000.0, 20_000.0, 50_000.0, ]); diff --git a/core/node/node_sync/src/tree_data_fetcher/mod.rs b/core/node/node_sync/src/tree_data_fetcher/mod.rs index d155e03b5563..c871ec16b9de 100644 --- a/core/node/node_sync/src/tree_data_fetcher/mod.rs +++ b/core/node/node_sync/src/tree_data_fetcher/mod.rs @@ -22,6 +22,7 @@ use self::{ metrics::{ProcessingStage, TreeDataFetcherMetrics, METRICS}, provider::{L1DataProvider, MissingData, TreeDataProvider}, }; +use crate::tree_data_fetcher::provider::CombinedDataProvider; mod metrics; mod provider; @@ -30,7 +31,7 @@ mod tests; #[derive(Debug, thiserror::Error)] pub(crate) enum TreeDataFetcherError { - #[error("error fetching data from main node")] + #[error("error fetching data")] Rpc(#[from] EnrichedClientError), #[error("internal error")] Internal(#[from] anyhow::Error), @@ -95,7 +96,7 @@ enum StepOutcome { /// by Consistency checker. #[derive(Debug)] pub struct TreeDataFetcher { - data_provider: Box, + data_provider: CombinedDataProvider, // Used in the Info metric diamond_proxy_address: Option
, pool: ConnectionPool, @@ -112,7 +113,7 @@ impl TreeDataFetcher { /// Creates a new fetcher connected to the main node. pub fn new(client: Box>, pool: ConnectionPool) -> Self { Self { - data_provider: Box::new(client.for_component("tree_data_fetcher")), + data_provider: CombinedDataProvider::new(client.for_component("tree_data_fetcher")), diamond_proxy_address: None, pool, metrics: &METRICS, @@ -140,7 +141,7 @@ impl TreeDataFetcher { eth_client.for_component("tree_data_fetcher"), diamond_proxy_address, )?; - self.data_provider = Box::new(l1_provider.with_fallback(self.data_provider)); + self.data_provider.set_l1(l1_provider); self.diamond_proxy_address = Some(diamond_proxy_address); Ok(self) } @@ -212,14 +213,11 @@ impl TreeDataFetcher { .await?; stage_latency.observe(); let root_hash = match root_hash_result { - Ok(output) => { + Ok(root_hash) => { tracing::debug!( - "Received root hash for L1 batch #{l1_batch_to_fetch} from {source:?}: {root_hash:?}", - source = output.source, - root_hash = output.root_hash + "Received root hash for L1 batch #{l1_batch_to_fetch}: {root_hash:?}" ); - self.metrics.root_hash_sources[&output.source].inc(); - output.root_hash + root_hash } Err(MissingData::Batch) => { let err = anyhow::anyhow!( diff --git a/core/node/node_sync/src/tree_data_fetcher/provider/mod.rs b/core/node/node_sync/src/tree_data_fetcher/provider/mod.rs index 0c9362369fe6..867ea2427541 100644 --- a/core/node/node_sync/src/tree_data_fetcher/provider/mod.rs +++ b/core/node/node_sync/src/tree_data_fetcher/provider/mod.rs @@ -2,7 +2,6 @@ use std::fmt; use anyhow::Context; use async_trait::async_trait; -use vise::{EncodeLabelSet, EncodeLabelValue}; use zksync_eth_client::EthInterface; use zksync_types::{block::L2BlockHeader, web3, Address, L1BatchNumber, H256, U256, U64}; use zksync_web3_decl::{ @@ -12,7 +11,10 @@ use zksync_web3_decl::{ namespaces::ZksNamespaceClient, }; -use super::{metrics::METRICS, TreeDataFetcherResult}; +use super::{ + metrics::{ProcessingStage, TreeDataProviderSource, METRICS}, + TreeDataFetcherResult, +}; #[cfg(test)] mod tests; @@ -29,21 +31,7 @@ pub(super) enum MissingData { PossibleReorg, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)] -#[metrics(label = "source", rename_all = "snake_case")] -pub(super) enum TreeDataProviderSource { - L1CommitEvent, - BatchDetailsRpc, -} - -#[derive(Debug)] -pub(super) struct TreeDataProviderOutput { - pub root_hash: H256, - pub source: TreeDataProviderSource, -} - -pub(super) type TreeDataProviderResult = - TreeDataFetcherResult>; +pub(super) type TreeDataProviderResult = TreeDataFetcherResult>; /// External provider of tree data, such as main node (via JSON-RPC). #[async_trait] @@ -92,14 +80,7 @@ impl TreeDataProvider for Box> { return Ok(Err(MissingData::PossibleReorg)); } - Ok(batch_details - .base - .root_hash - .ok_or(MissingData::RootHash) - .map(|root_hash| TreeDataProviderOutput { - root_hash, - source: TreeDataProviderSource::BatchDetailsRpc, - })) + Ok(batch_details.base.root_hash.ok_or(MissingData::RootHash)) } } @@ -205,13 +186,6 @@ impl L1DataProvider { })?; Ok((number, block.timestamp)) } - - pub fn with_fallback(self, fallback: Box) -> CombinedDataProvider { - CombinedDataProvider { - l1: Some(self), - fallback, - } - } } #[async_trait] @@ -305,10 +279,7 @@ impl TreeDataProvider for L1DataProvider { l1_commit_block_number, l1_commit_block_timestamp: l1_commit_block.timestamp, }); - Ok(Ok(TreeDataProviderOutput { - root_hash, - source: TreeDataProviderSource::L1CommitEvent, - })) + Ok(Ok(root_hash)) } _ => { tracing::warn!( @@ -325,44 +296,69 @@ impl TreeDataProvider for L1DataProvider { #[derive(Debug)] pub(super) struct CombinedDataProvider { l1: Option, - fallback: Box, + // Generic to allow for tests. + rpc: Box, +} + +impl CombinedDataProvider { + pub fn new(fallback: impl TreeDataProvider) -> Self { + Self { + l1: None, + rpc: Box::new(fallback), + } + } + + pub fn set_l1(&mut self, l1: L1DataProvider) { + self.l1 = Some(l1); + } } #[async_trait] impl TreeDataProvider for CombinedDataProvider { + #[tracing::instrument(skip(self, last_l2_block))] async fn batch_details( &mut self, number: L1BatchNumber, last_l2_block: &L2BlockHeader, ) -> TreeDataProviderResult { if let Some(l1) = &mut self.l1 { - match l1.batch_details(number, last_l2_block).await { + let stage_latency = METRICS.stage_latency[&ProcessingStage::FetchL1CommitEvent].start(); + let l1_result = l1.batch_details(number, last_l2_block).await; + stage_latency.observe(); + + match l1_result { Err(err) => { if err.is_transient() { tracing::info!( - number = number.0, - "Transient error calling L1 data provider: {err}" + "Transient error calling L1 data provider: {:#}", + anyhow::Error::from(err) ); } else { tracing::warn!( - number = number.0, - "Fatal error calling L1 data provider: {err}" + "Fatal error calling L1 data provider: {:#}", + anyhow::Error::from(err) ); self.l1 = None; } } - Ok(Ok(root_hash)) => return Ok(Ok(root_hash)), + Ok(Ok(root_hash)) => { + METRICS.root_hash_sources[&TreeDataProviderSource::L1CommitEvent].inc(); + return Ok(Ok(root_hash)); + } Ok(Err(missing_data)) => { - tracing::debug!( - number = number.0, - "L1 data provider misses batch data: {missing_data}" - ); + tracing::info!("L1 data provider misses batch data: {missing_data}"); // No sense of calling the L1 provider in the future; the L2 provider will very likely get information // about batches significantly faster. self.l1 = None; } } } - self.fallback.batch_details(number, last_l2_block).await + let stage_latency = METRICS.stage_latency[&ProcessingStage::FetchBatchDetailsRpc].start(); + let rpc_result = self.rpc.batch_details(number, last_l2_block).await; + stage_latency.observe(); + if matches!(rpc_result, Ok(Ok(_))) { + METRICS.root_hash_sources[&TreeDataProviderSource::BatchDetailsRpc].inc(); + } + rpc_result } } diff --git a/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs b/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs index 291bc71c897d..09fa16f16077 100644 --- a/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs +++ b/core/node/node_sync/src/tree_data_fetcher/provider/tests.rs @@ -86,13 +86,12 @@ async fn rpc_data_provider_basics() { }; let mut client: Box> = Box::new(l2_parameters.mock_client()); - let output = client + let root_hash = client .batch_details(L1BatchNumber(1), &last_l2_block) .await .unwrap() .expect("missing block"); - assert_eq!(output.root_hash, H256::from_low_u64_be(1)); - assert_matches!(output.source, TreeDataProviderSource::BatchDetailsRpc); + assert_eq!(root_hash, H256::from_low_u64_be(1)); // Query a future L1 batch. let output = client @@ -270,13 +269,12 @@ async fn test_using_l1_data_provider(l1_batch_timestamps: &[u64]) { L1DataProvider::new(Box::new(eth_params.client()), DIAMOND_PROXY_ADDRESS).unwrap(); for i in 0..l1_batch_timestamps.len() { let number = L1BatchNumber(i as u32 + 1); - let output = provider + let root_hash = provider .batch_details(number, &get_last_l2_block(&mut storage, number).await) .await .unwrap() .expect("no root hash"); - assert_eq!(output.root_hash, H256::repeat_byte(number.0 as u8)); - assert_matches!(output.source, TreeDataProviderSource::L1CommitEvent); + assert_eq!(root_hash, H256::repeat_byte(number.0 as u8)); let past_l1_batch = provider.past_l1_batch.unwrap(); assert_eq!(past_l1_batch.number, number); @@ -352,12 +350,13 @@ async fn combined_data_provider_errors() { let mut main_node_client = MockMainNodeClient::default(); main_node_client.insert_batch(L1BatchNumber(2), H256::repeat_byte(2)); - let mut provider = L1DataProvider::new(Box::new(eth_params.client()), DIAMOND_PROXY_ADDRESS) - .unwrap() - .with_fallback(Box::new(main_node_client)); + let mut provider = CombinedDataProvider::new(main_node_client); + let l1_provider = + L1DataProvider::new(Box::new(eth_params.client()), DIAMOND_PROXY_ADDRESS).unwrap(); + provider.set_l1(l1_provider); // L1 batch #1 should be obtained from L1 - let output = provider + let root_hash = provider .batch_details( L1BatchNumber(1), &get_last_l2_block(&mut storage, L1BatchNumber(1)).await, @@ -365,12 +364,11 @@ async fn combined_data_provider_errors() { .await .unwrap() .expect("no root hash"); - assert_eq!(output.root_hash, H256::repeat_byte(1)); - assert_matches!(output.source, TreeDataProviderSource::L1CommitEvent); + assert_eq!(root_hash, H256::repeat_byte(1)); assert!(provider.l1.is_some()); // L1 batch #2 should be obtained from L2 - let output = provider + let root_hash = provider .batch_details( L1BatchNumber(2), &get_last_l2_block(&mut storage, L1BatchNumber(2)).await, @@ -378,7 +376,6 @@ async fn combined_data_provider_errors() { .await .unwrap() .expect("no root hash"); - assert_eq!(output.root_hash, H256::repeat_byte(2)); - assert_matches!(output.source, TreeDataProviderSource::BatchDetailsRpc); + assert_eq!(root_hash, H256::repeat_byte(2)); assert!(provider.l1.is_none()); } diff --git a/core/node/node_sync/src/tree_data_fetcher/tests.rs b/core/node/node_sync/src/tree_data_fetcher/tests.rs index 3ffbb91d474a..5d94ddf658d6 100644 --- a/core/node/node_sync/src/tree_data_fetcher/tests.rs +++ b/core/node/node_sync/src/tree_data_fetcher/tests.rs @@ -16,11 +16,7 @@ use zksync_node_test_utils::{create_l1_batch, create_l2_block, prepare_recovery_ use zksync_types::{AccountTreeId, Address, L2BlockNumber, StorageKey, StorageLog, H256}; use zksync_web3_decl::jsonrpsee::core::ClientError; -use super::{ - metrics::StepOutcomeLabel, - provider::{TreeDataProviderOutput, TreeDataProviderResult, TreeDataProviderSource}, - *, -}; +use super::{metrics::StepOutcomeLabel, provider::TreeDataProviderResult, *}; #[derive(Debug, Default)] pub(super) struct MockMainNodeClient { @@ -48,10 +44,7 @@ impl TreeDataProvider for MockMainNodeClient { Ok(self .batch_details_responses .get(&number) - .map(|&root_hash| TreeDataProviderOutput { - root_hash, - source: TreeDataProviderSource::BatchDetailsRpc, - }) + .copied() .ok_or(MissingData::Batch)) } } @@ -122,7 +115,7 @@ impl FetcherHarness { let (updates_sender, updates_receiver) = mpsc::unbounded_channel(); let metrics = &*Box::leak(Box::::default()); let fetcher = TreeDataFetcher { - data_provider: Box::new(client), + data_provider: CombinedDataProvider::new(client), diamond_proxy_address: None, pool: pool.clone(), metrics, @@ -324,10 +317,7 @@ impl TreeDataProvider for SlowMainNode { } let request_count = self.request_count.fetch_add(1, Ordering::Relaxed); Ok(if request_count >= self.compute_root_hash_after { - Ok(TreeDataProviderOutput { - root_hash: H256::repeat_byte(1), - source: TreeDataProviderSource::BatchDetailsRpc, - }) + Ok(H256::repeat_byte(1)) } else { Err(MissingData::RootHash) }) From cb6a6c88de54806d0f4ae4af7ea873a911605780 Mon Sep 17 00:00:00 2001 From: pompon0 Date: Wed, 19 Jun 2024 10:17:28 +0200 Subject: [PATCH 18/43] feat: upgraded encoding of transactions in consensus Payload. (#2245) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently the encoded transaction is up to 3x larger than its rlp encoding. This is caused by data duplication between: raw, input and factory_deps fields. In the new encoding we use the rlp directly. It will be used starting with protocol version 25. --------- Co-authored-by: Bruno França --- Cargo.lock | 4 + core/lib/basic_types/src/protocol_version.rs | 4 +- core/lib/dal/Cargo.toml | 4 + core/lib/dal/src/consensus/mod.rs | 118 +++++++++++++++---- core/lib/dal/src/consensus/proto/mod.proto | 20 ++++ core/lib/dal/src/consensus/tests.rs | 64 +++++++++- core/node/consensus/Cargo.toml | 1 + core/node/consensus/src/storage/mod.rs | 2 +- core/node/consensus/src/storage/testonly.rs | 44 ++++++- core/node/consensus/src/testonly.rs | 14 ++- core/node/consensus/src/tests.rs | 71 ++++++----- core/node/test_utils/src/lib.rs | 13 +- core/tests/test_account/Cargo.toml | 1 + core/tests/test_account/src/lib.rs | 4 + prover/Cargo.lock | 1 + 15 files changed, 289 insertions(+), 76 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ccfb6715884e..c41faf9d1faf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8480,6 +8480,7 @@ dependencies = [ "tokio", "tracing", "vise", + "zksync_concurrency", "zksync_consensus_roles", "zksync_consensus_storage", "zksync_contracts", @@ -8487,6 +8488,7 @@ dependencies = [ "zksync_protobuf", "zksync_protobuf_build", "zksync_system_constants", + "zksync_test_account", "zksync_types", "zksync_utils", ] @@ -8857,6 +8859,7 @@ dependencies = [ "zksync_consensus_roles", "zksync_consensus_storage", "zksync_consensus_utils", + "zksync_contracts", "zksync_dal", "zksync_l1_contract_interface", "zksync_merkle_tree", @@ -9387,6 +9390,7 @@ version = "0.1.0" dependencies = [ "ethabi", "hex", + "rand 0.8.5", "zksync_contracts", "zksync_eth_signer", "zksync_system_constants", diff --git a/core/lib/basic_types/src/protocol_version.rs b/core/lib/basic_types/src/protocol_version.rs index d8083c0f6a31..f0d12436e3b8 100644 --- a/core/lib/basic_types/src/protocol_version.rs +++ b/core/lib/basic_types/src/protocol_version.rs @@ -71,11 +71,11 @@ pub enum ProtocolVersionId { } impl ProtocolVersionId { - pub fn latest() -> Self { + pub const fn latest() -> Self { Self::Version24 } - pub fn next() -> Self { + pub const fn next() -> Self { Self::Version25 } diff --git a/core/lib/dal/Cargo.toml b/core/lib/dal/Cargo.toml index 034f252f7e50..aa1d7097b9ba 100644 --- a/core/lib/dal/Cargo.toml +++ b/core/lib/dal/Cargo.toml @@ -49,5 +49,9 @@ strum = { workspace = true, features = ["derive"] } tracing.workspace = true chrono = { workspace = true, features = ["serde"] } +[dev-dependencies] +zksync_test_account.workspace = true +zksync_concurrency.workspace = true + [build-dependencies] zksync_protobuf_build.workspace = true diff --git a/core/lib/dal/src/consensus/mod.rs b/core/lib/dal/src/consensus/mod.rs index 8e1f246b657c..fac045ce2224 100644 --- a/core/lib/dal/src/consensus/mod.rs +++ b/core/lib/dal/src/consensus/mod.rs @@ -7,6 +7,7 @@ use anyhow::{anyhow, Context as _}; use zksync_consensus_roles::validator; use zksync_protobuf::{required, ProtoFmt, ProtoRepr}; use zksync_types::{ + abi, ethabi, fee::Fee, l1::{OpProcessingType, PriorityQueueType}, l2::TransactionType, @@ -38,38 +39,59 @@ pub struct Payload { impl ProtoFmt for Payload { type Proto = proto::Payload; - fn read(message: &Self::Proto) -> anyhow::Result { - let mut transactions = Vec::with_capacity(message.transactions.len()); - for (i, tx) in message.transactions.iter().enumerate() { - transactions.push(tx.read().with_context(|| format!("transactions[{i}]"))?) + fn read(r: &Self::Proto) -> anyhow::Result { + let protocol_version = required(&r.protocol_version) + .and_then(|x| Ok(ProtocolVersionId::try_from(u16::try_from(*x)?)?)) + .context("protocol_version")?; + let mut transactions = vec![]; + + match protocol_version { + v if v >= ProtocolVersionId::Version25 => { + anyhow::ensure!( + r.transactions.is_empty(), + "transactions should be empty in protocol_version {v}" + ); + for (i, tx) in r.transactions_v25.iter().enumerate() { + transactions.push( + tx.read() + .with_context(|| format!("transactions_v25[{i}]"))?, + ); + } + } + v => { + anyhow::ensure!( + r.transactions_v25.is_empty(), + "transactions_v25 should be empty in protocol_version {v}" + ); + for (i, tx) in r.transactions.iter().enumerate() { + transactions.push(tx.read().with_context(|| format!("transactions[{i}]"))?) + } + } } Ok(Self { - protocol_version: required(&message.protocol_version) - .and_then(|x| Ok(ProtocolVersionId::try_from(u16::try_from(*x)?)?)) - .context("protocol_version")?, - hash: required(&message.hash) + protocol_version, + hash: required(&r.hash) .and_then(|h| parse_h256(h)) .context("hash")?, l1_batch_number: L1BatchNumber( - *required(&message.l1_batch_number).context("l1_batch_number")?, + *required(&r.l1_batch_number).context("l1_batch_number")?, ), - timestamp: *required(&message.timestamp).context("timestamp")?, - l1_gas_price: *required(&message.l1_gas_price).context("l1_gas_price")?, - l2_fair_gas_price: *required(&message.l2_fair_gas_price) - .context("l2_fair_gas_price")?, - fair_pubdata_price: message.fair_pubdata_price, - virtual_blocks: *required(&message.virtual_blocks).context("virtual_blocks")?, - operator_address: required(&message.operator_address) + timestamp: *required(&r.timestamp).context("timestamp")?, + l1_gas_price: *required(&r.l1_gas_price).context("l1_gas_price")?, + l2_fair_gas_price: *required(&r.l2_fair_gas_price).context("l2_fair_gas_price")?, + fair_pubdata_price: r.fair_pubdata_price, + virtual_blocks: *required(&r.virtual_blocks).context("virtual_blocks")?, + operator_address: required(&r.operator_address) .and_then(|a| parse_h160(a)) .context("operator_address")?, transactions, - last_in_batch: *required(&message.last_in_batch).context("last_in_batch")?, + last_in_batch: *required(&r.last_in_batch).context("last_in_batch")?, }) } fn build(&self) -> Self::Proto { - Self::Proto { + let mut x = Self::Proto { protocol_version: Some((self.protocol_version as u16).into()), hash: Some(self.hash.as_bytes().into()), l1_batch_number: Some(self.l1_batch_number.0), @@ -80,13 +102,19 @@ impl ProtoFmt for Payload { virtual_blocks: Some(self.virtual_blocks), operator_address: Some(self.operator_address.as_bytes().into()), // Transactions are stored in execution order, therefore order is deterministic. - transactions: self - .transactions - .iter() - .map(proto::Transaction::build) - .collect(), + transactions: vec![], + transactions_v25: vec![], last_in_batch: Some(self.last_in_batch), + }; + match self.protocol_version { + v if v >= ProtocolVersionId::Version25 => { + x.transactions_v25 = self.transactions.iter().map(ProtoRepr::build).collect(); + } + _ => { + x.transactions = self.transactions.iter().map(ProtoRepr::build).collect(); + } } + x } } @@ -100,6 +128,50 @@ impl Payload { } } +impl ProtoRepr for proto::TransactionV25 { + type Type = Transaction; + + fn read(&self) -> anyhow::Result { + use proto::transaction_v25::T; + let tx = match required(&self.t)? { + T::L1(l1) => abi::Transaction::L1 { + tx: required(&l1.rlp) + .and_then(|x| { + let tokens = ethabi::decode(&[abi::L2CanonicalTransaction::schema()], x) + .context("ethabi::decode()")?; + // Unwrap is safe because `ethabi::decode` does the verification. + let tx = + abi::L2CanonicalTransaction::decode(tokens.into_iter().next().unwrap()) + .context("L2CanonicalTransaction::decode()")?; + Ok(tx) + }) + .context("rlp")? + .into(), + factory_deps: l1.factory_deps.clone(), + eth_block: 0, + }, + T::L2(l2) => abi::Transaction::L2(required(&l2.rlp).context("rlp")?.clone()), + }; + tx.try_into() + } + + fn build(tx: &Self::Type) -> Self { + let tx = abi::Transaction::try_from(tx.clone()).unwrap(); + use proto::transaction_v25::T; + Self { + t: Some(match tx { + abi::Transaction::L1 { + tx, factory_deps, .. + } => T::L1(proto::L1Transaction { + rlp: Some(ethabi::encode(&[tx.encode()])), + factory_deps, + }), + abi::Transaction::L2(tx) => T::L2(proto::L2Transaction { rlp: Some(tx) }), + }), + } + } +} + impl ProtoRepr for proto::Transaction { type Type = Transaction; diff --git a/core/lib/dal/src/consensus/proto/mod.proto b/core/lib/dal/src/consensus/proto/mod.proto index a53647611836..a7b5ea344152 100644 --- a/core/lib/dal/src/consensus/proto/mod.proto +++ b/core/lib/dal/src/consensus/proto/mod.proto @@ -13,10 +13,30 @@ message Payload { optional uint64 fair_pubdata_price = 11; // required since 1.4.1; gwei optional uint32 virtual_blocks = 6; // required optional bytes operator_address = 7; // required; H160 + // Set for protocol_version < 25. repeated Transaction transactions = 8; + // Set for protocol_version >= 25. + repeated TransactionV25 transactions_v25 = 12; optional bool last_in_batch = 10; // required } +message L1Transaction { + optional bytes rlp = 1; // required; RLP encoded L2CanonicalTransaction + repeated bytes factory_deps = 2; +} + +message L2Transaction { + optional bytes rlp = 1; // required; RLP encoded TransactionRequest +} + +message TransactionV25 { + // required + oneof t { + L1Transaction l1 = 1; + L2Transaction l2 = 2; + } +} + message Transaction { reserved 5; reserved "received_timestamp_ms"; diff --git a/core/lib/dal/src/consensus/tests.rs b/core/lib/dal/src/consensus/tests.rs index 694634f11a8c..4a69bebdc362 100644 --- a/core/lib/dal/src/consensus/tests.rs +++ b/core/lib/dal/src/consensus/tests.rs @@ -1,21 +1,75 @@ use std::fmt::Debug; +use rand::Rng; +use zksync_concurrency::ctx; use zksync_protobuf::{ repr::{decode, encode}, + testonly::test_encode, ProtoRepr, }; -use zksync_types::{web3::Bytes, Execute, ExecuteTransactionCommon, Transaction}; +use zksync_test_account::Account; +use zksync_types::{ + web3::Bytes, Execute, ExecuteTransactionCommon, L1BatchNumber, ProtocolVersionId, Transaction, +}; + +use super::{proto, Payload}; +use crate::tests::mock_protocol_upgrade_transaction; + +fn execute(rng: &mut impl Rng) -> Execute { + Execute { + contract_address: rng.gen(), + value: rng.gen::().into(), + calldata: (0..10 * 32).map(|_| rng.gen()).collect(), + // TODO: find a way to generate valid random bytecode. + factory_deps: vec![], + } +} -use crate::tests::{mock_l1_execute, mock_l2_transaction, mock_protocol_upgrade_transaction}; +fn l1_transaction(rng: &mut impl Rng) -> Transaction { + Account::random_using(rng).get_l1_tx(execute(rng), rng.gen()) +} + +fn l2_transaction(rng: &mut impl Rng) -> Transaction { + Account::random_using(rng).get_l2_tx_for_execute(execute(rng), None) +} + +fn payload(rng: &mut impl Rng, protocol_version: ProtocolVersionId) -> Payload { + Payload { + protocol_version, + hash: rng.gen(), + l1_batch_number: L1BatchNumber(rng.gen()), + timestamp: rng.gen(), + l1_gas_price: rng.gen(), + l2_fair_gas_price: rng.gen(), + fair_pubdata_price: Some(rng.gen()), + virtual_blocks: rng.gen(), + operator_address: rng.gen(), + transactions: (0..10) + .map(|_| match rng.gen() { + true => l1_transaction(rng), + false => l2_transaction(rng), + }) + .collect(), + last_in_batch: rng.gen(), + } +} /// Tests struct <-> proto struct conversions. #[test] fn test_encoding() { - encode_decode::(mock_l1_execute().into()); - encode_decode::(mock_l2_transaction().into()); - encode_decode::( + let ctx = &ctx::test_root(&ctx::RealClock); + let rng = &mut ctx.rng(); + encode_decode::(l1_transaction(rng)); + encode_decode::(l2_transaction(rng)); + encode_decode::(l1_transaction(rng)); + encode_decode::(l2_transaction(rng)); + encode_decode::( mock_protocol_upgrade_transaction().into(), ); + let p = payload(rng, ProtocolVersionId::Version24); + test_encode(rng, &p); + let p = payload(rng, ProtocolVersionId::Version25); + test_encode(rng, &p); } fn encode_decode(msg: P::Type) diff --git a/core/node/consensus/Cargo.toml b/core/node/consensus/Cargo.toml index b22fde34e7c6..5fc95b6c91f3 100644 --- a/core/node/consensus/Cargo.toml +++ b/core/node/consensus/Cargo.toml @@ -43,6 +43,7 @@ zksync_node_genesis.workspace = true zksync_node_test_utils.workspace = true zksync_node_api_server.workspace = true zksync_test_account.workspace = true +zksync_contracts.workspace= true tokio.workspace = true test-casing.workspace = true diff --git a/core/node/consensus/src/storage/mod.rs b/core/node/consensus/src/storage/mod.rs index cf45f89ad11e..bc8a0b8b8409 100644 --- a/core/node/consensus/src/storage/mod.rs +++ b/core/node/consensus/src/storage/mod.rs @@ -18,7 +18,7 @@ use zksync_types::{commitment::L1BatchWithMetadata, L1BatchNumber, L2BlockNumber use super::config; #[cfg(test)] -mod testonly; +pub(crate) mod testonly; /// Context-aware `zksync_dal::ConnectionPool` wrapper. #[derive(Debug, Clone)] diff --git a/core/node/consensus/src/storage/testonly.rs b/core/node/consensus/src/storage/testonly.rs index ccac1f7e45a9..f5f30021b7c4 100644 --- a/core/node/consensus/src/storage/testonly.rs +++ b/core/node/consensus/src/storage/testonly.rs @@ -3,13 +3,49 @@ use anyhow::Context as _; use zksync_concurrency::{ctx, error::Wrap as _, time}; use zksync_consensus_roles::validator; -use zksync_node_genesis::{insert_genesis_batch, GenesisParams}; +use zksync_contracts::BaseSystemContracts; +use zksync_node_genesis::{insert_genesis_batch, mock_genesis_config, GenesisParams}; use zksync_node_test_utils::{recover, snapshot, Snapshot}; -use zksync_types::{commitment::L1BatchWithMetadata, L1BatchNumber}; +use zksync_types::{ + commitment::L1BatchWithMetadata, protocol_version::ProtocolSemanticVersion, + system_contracts::get_system_smart_contracts, L1BatchNumber, L2BlockNumber, ProtocolVersionId, +}; use super::ConnectionPool; +pub(crate) fn mock_genesis_params(protocol_version: ProtocolVersionId) -> GenesisParams { + let mut cfg = mock_genesis_config(); + cfg.protocol_version = Some(ProtocolSemanticVersion { + minor: protocol_version, + patch: 0.into(), + }); + GenesisParams::from_genesis_config( + cfg, + BaseSystemContracts::load_from_disk(), + get_system_smart_contracts(), + ) + .unwrap() +} + impl ConnectionPool { + pub(crate) async fn test( + from_snapshot: bool, + protocol_version: ProtocolVersionId, + ) -> ConnectionPool { + match from_snapshot { + true => { + ConnectionPool::from_snapshot(Snapshot::make( + L1BatchNumber(23), + L2BlockNumber(87), + &[], + mock_genesis_params(protocol_version), + )) + .await + } + false => ConnectionPool::from_genesis(protocol_version).await, + } + } + /// Waits for the `number` L2 block to have a certificate. pub async fn wait_for_certificate( &self, @@ -60,11 +96,11 @@ impl ConnectionPool { } /// Constructs a new db initialized with genesis state. - pub(crate) async fn from_genesis() -> Self { + pub(crate) async fn from_genesis(protocol_version: ProtocolVersionId) -> Self { let pool = zksync_dal::ConnectionPool::test_pool().await; { let mut storage = pool.connection().await.unwrap(); - insert_genesis_batch(&mut storage, &GenesisParams::mock()) + insert_genesis_batch(&mut storage, &mock_genesis_params(protocol_version)) .await .unwrap(); } diff --git a/core/node/consensus/src/testonly.rs b/core/node/consensus/src/testonly.rs index 5baa1c7b1eed..ce16efed2225 100644 --- a/core/node/consensus/src/testonly.rs +++ b/core/node/consensus/src/testonly.rs @@ -54,6 +54,7 @@ use crate::{ /// Fake StateKeeper for tests. pub(super) struct StateKeeper { + protocol_version: ProtocolVersionId, // Batch of the `last_block`. last_batch: L1BatchNumber, last_block: L2BlockNumber, @@ -130,6 +131,16 @@ impl StateKeeper { pool: ConnectionPool, ) -> ctx::Result<(Self, StateKeeperRunner)> { let mut conn = pool.connection(ctx).await.wrap("connection()")?; + // We fetch the last protocol version from storage. + // `protocol_version_id_by_timestamp` does a wrapping conversion to `i64`. + let protocol_version = ctx + .wait( + conn.0 + .protocol_versions_dal() + .protocol_version_id_by_timestamp(i64::MAX.try_into().unwrap()), + ) + .await? + .context("protocol_version_id_by_timestamp()")?; let cursor = ctx .wait(IoCursor::for_fetcher(&mut conn.0)) .await? @@ -164,6 +175,7 @@ impl StateKeeper { let account = Account::random(); Ok(( Self { + protocol_version, last_batch: cursor.l1_batch, last_block: cursor.next_l2_block - 1, last_timestamp: cursor.prev_l2_block_timestamp, @@ -196,7 +208,7 @@ impl StateKeeper { self.batch_sealed = false; SyncAction::OpenBatch { params: L1BatchParams { - protocol_version: ProtocolVersionId::latest(), + protocol_version: self.protocol_version, validation_computational_gas_limit: u32::MAX, operator_address: GenesisParams::mock().config().fee_account, fee_input: BatchFeeInput::L1Pegged(L1PeggedBatchFeeModelInput { diff --git a/core/node/consensus/src/tests.rs b/core/node/consensus/src/tests.rs index 79784f0fbb51..b16c66e478bb 100644 --- a/core/node/consensus/src/tests.rs +++ b/core/node/consensus/src/tests.rs @@ -1,6 +1,6 @@ #![allow(unused)] use anyhow::Context as _; -use test_casing::test_casing; +use test_casing::{test_casing, Product}; use tracing::Instrument as _; use zksync_concurrency::{ctx, scope}; use zksync_config::configs::consensus::{ValidatorPublicKey, WeightedValidator}; @@ -12,26 +12,20 @@ use zksync_consensus_roles::{ }; use zksync_dal::CoreDal; use zksync_node_test_utils::Snapshot; -use zksync_types::{L1BatchNumber, L2BlockNumber}; +use zksync_types::{L1BatchNumber, L2BlockNumber, ProtocolVersionId}; use super::*; -async fn new_pool(from_snapshot: bool) -> ConnectionPool { - match from_snapshot { - true => { - ConnectionPool::from_snapshot(Snapshot::make(L1BatchNumber(23), L2BlockNumber(87), &[])) - .await - } - false => ConnectionPool::from_genesis().await, - } -} +const VERSIONS: [ProtocolVersionId; 2] = [ProtocolVersionId::latest(), ProtocolVersionId::next()]; +const FROM_SNAPSHOT: [bool; 2] = [true, false]; +#[test_casing(2, VERSIONS)] #[tokio::test(flavor = "multi_thread")] -async fn test_validator_block_store() { +async fn test_validator_block_store(version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); - let pool = new_pool(false).await; + let pool = ConnectionPool::test(false, version).await; // Fill storage with unsigned L2 blocks. // Fetch a suffix of blocks that we will generate (fake) certs for. @@ -91,9 +85,9 @@ async fn test_validator_block_store() { // In the current implementation, consensus certificates are created asynchronously // for the L2 blocks constructed by the StateKeeper. This means that consensus actor // is effectively just back filling the consensus certificates for the L2 blocks in storage. -#[test_casing(2, [false, true])] +#[test_casing(4, Product((FROM_SNAPSHOT,VERSIONS)))] #[tokio::test(flavor = "multi_thread")] -async fn test_validator(from_snapshot: bool) { +async fn test_validator(from_snapshot: bool, version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::AffineClock::new(10.)); let rng = &mut ctx.rng(); @@ -102,7 +96,7 @@ async fn test_validator(from_snapshot: bool) { scope::run!(ctx, |ctx, s| async { tracing::info!("Start state keeper."); - let pool = new_pool(from_snapshot).await; + let pool = ConnectionPool::test(from_snapshot,version).await; let (mut sk, runner) = testonly::StateKeeper::new(ctx, pool.clone()).await?; s.spawn_bg(runner.run(ctx)); @@ -155,8 +149,9 @@ async fn test_validator(from_snapshot: bool) { } // Test running a validator node and 2 full nodes recovered from different snapshots. +#[test_casing(2, VERSIONS)] #[tokio::test(flavor = "multi_thread")] -async fn test_nodes_from_various_snapshots() { +async fn test_nodes_from_various_snapshots(version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::AffineClock::new(10.)); let rng = &mut ctx.rng(); @@ -165,7 +160,7 @@ async fn test_nodes_from_various_snapshots() { scope::run!(ctx, |ctx, s| async { tracing::info!("spawn validator"); - let validator_pool = ConnectionPool::from_genesis().await; + let validator_pool = ConnectionPool::from_genesis(version).await; let (mut validator, runner) = testonly::StateKeeper::new(ctx, validator_pool.clone()).await?; s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("validator"))); @@ -233,9 +228,9 @@ async fn test_nodes_from_various_snapshots() { // Test running a validator node and a couple of full nodes. // Validator is producing signed blocks and fetchers are expected to fetch // them directly or indirectly. -#[test_casing(2, [false, true])] +#[test_casing(4, Product((FROM_SNAPSHOT,VERSIONS)))] #[tokio::test(flavor = "multi_thread")] -async fn test_full_nodes(from_snapshot: bool) { +async fn test_full_nodes(from_snapshot: bool, version: ProtocolVersionId) { const NODES: usize = 2; zksync_concurrency::testonly::abort_on_panic(); @@ -256,7 +251,7 @@ async fn test_full_nodes(from_snapshot: bool) { // Run validator and fetchers in parallel. scope::run!(ctx, |ctx, s| async { - let validator_pool = new_pool(from_snapshot).await; + let validator_pool = ConnectionPool::test(from_snapshot, version).await; let (mut validator, runner) = testonly::StateKeeper::new(ctx, validator_pool.clone()).await?; s.spawn_bg(async { @@ -272,8 +267,7 @@ async fn test_full_nodes(from_snapshot: bool) { validator.seal_batch().await; validator_pool .wait_for_payload(ctx, validator.last_block()) - .await - .unwrap(); + .await?; tracing::info!("Run validator."); let (cfg, secrets) = testonly::config(&validator_cfgs[0]); @@ -283,7 +277,7 @@ async fn test_full_nodes(from_snapshot: bool) { let mut node_pools = vec![]; for (i, cfg) in node_cfgs.iter().enumerate() { let i = ctx::NoCopy(i); - let pool = new_pool(from_snapshot).await; + let pool = ConnectionPool::test(from_snapshot, version).await; let (node, runner) = testonly::StateKeeper::new(ctx, pool.clone()).await?; node_pools.push(pool.clone()); s.spawn_bg(async { @@ -318,9 +312,9 @@ async fn test_full_nodes(from_snapshot: bool) { } // Test running external node (non-leader) validators. -#[test_casing(2, [false, true])] +#[test_casing(4, Product((FROM_SNAPSHOT,VERSIONS)))] #[tokio::test(flavor = "multi_thread")] -async fn test_en_validators(from_snapshot: bool) { +async fn test_en_validators(from_snapshot: bool, version: ProtocolVersionId) { const NODES: usize = 3; zksync_concurrency::testonly::abort_on_panic(); @@ -331,7 +325,7 @@ async fn test_en_validators(from_snapshot: bool) { // Run all nodes in parallel. scope::run!(ctx, |ctx, s| async { - let main_node_pool = new_pool(from_snapshot).await; + let main_node_pool = ConnectionPool::test(from_snapshot, version).await; let (mut main_node, runner) = testonly::StateKeeper::new(ctx, main_node_pool.clone()).await?; s.spawn_bg(async { @@ -370,7 +364,7 @@ async fn test_en_validators(from_snapshot: bool) { let mut ext_node_pools = vec![]; for (i, cfg) in cfgs[1..].iter().enumerate() { let i = ctx::NoCopy(i); - let pool = new_pool(from_snapshot).await; + let pool = ConnectionPool::test(from_snapshot, version).await; let (ext_node, runner) = testonly::StateKeeper::new(ctx, pool.clone()).await?; ext_node_pools.push(pool.clone()); s.spawn_bg(async { @@ -404,9 +398,9 @@ async fn test_en_validators(from_snapshot: bool) { } // Test fetcher back filling missing certs. -#[test_casing(2, [false, true])] +#[test_casing(4, Product((FROM_SNAPSHOT,VERSIONS)))] #[tokio::test(flavor = "multi_thread")] -async fn test_p2p_fetcher_backfill_certs(from_snapshot: bool) { +async fn test_p2p_fetcher_backfill_certs(from_snapshot: bool, version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::AffineClock::new(10.)); let rng = &mut ctx.rng(); @@ -416,7 +410,7 @@ async fn test_p2p_fetcher_backfill_certs(from_snapshot: bool) { scope::run!(ctx, |ctx, s| async { tracing::info!("Spawn validator."); - let validator_pool = new_pool(from_snapshot).await; + let validator_pool = ConnectionPool::test(from_snapshot, version).await; let (mut validator, runner) = testonly::StateKeeper::new(ctx, validator_pool.clone()).await?; s.spawn_bg(runner.run(ctx)); @@ -426,7 +420,7 @@ async fn test_p2p_fetcher_backfill_certs(from_snapshot: bool) { validator.seal_batch().await; let client = validator.connect(ctx).await?; - let node_pool = new_pool(from_snapshot).await; + let node_pool = ConnectionPool::test(from_snapshot, version).await; tracing::info!("Run p2p fetcher."); scope::run!(ctx, |ctx, s| async { @@ -479,16 +473,16 @@ async fn test_p2p_fetcher_backfill_certs(from_snapshot: bool) { .unwrap(); } -#[test_casing(2, [false, true])] +#[test_casing(4, Product((FROM_SNAPSHOT,VERSIONS)))] #[tokio::test] -async fn test_centralized_fetcher(from_snapshot: bool) { +async fn test_centralized_fetcher(from_snapshot: bool, version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); scope::run!(ctx, |ctx, s| async { tracing::info!("Spawn a validator."); - let validator_pool = new_pool(from_snapshot).await; + let validator_pool = ConnectionPool::test(from_snapshot, version).await; let (mut validator, runner) = testonly::StateKeeper::new(ctx, validator_pool.clone()).await?; s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("validator"))); @@ -498,7 +492,7 @@ async fn test_centralized_fetcher(from_snapshot: bool) { validator.seal_batch().await; tracing::info!("Spawn a node."); - let node_pool = new_pool(from_snapshot).await; + let node_pool = ConnectionPool::test(from_snapshot, version).await; let (node, runner) = testonly::StateKeeper::new(ctx, node_pool.clone()).await?; s.spawn_bg(runner.run(ctx).instrument(tracing::info_span!("fetcher"))); s.spawn_bg(node.run_fetcher(ctx, validator.connect(ctx).await?)); @@ -520,14 +514,15 @@ async fn test_centralized_fetcher(from_snapshot: bool) { /// Tests that generated L1 batch witnesses can be verified successfully. /// TODO: add tests for verification failures. +#[test_casing(2, VERSIONS)] #[tokio::test] -async fn test_batch_witness() { +async fn test_batch_witness(version: ProtocolVersionId) { zksync_concurrency::testonly::abort_on_panic(); let ctx = &ctx::test_root(&ctx::RealClock); let rng = &mut ctx.rng(); scope::run!(ctx, |ctx, s| async { - let pool = ConnectionPool::from_genesis().await; + let pool = ConnectionPool::from_genesis(version).await; let (mut node, runner) = testonly::StateKeeper::new(ctx, pool.clone()).await?; s.spawn_bg(runner.run_real(ctx)); diff --git a/core/node/test_utils/src/lib.rs b/core/node/test_utils/src/lib.rs index 566eab9c3d21..d0dfe367c21d 100644 --- a/core/node/test_utils/src/lib.rs +++ b/core/node/test_utils/src/lib.rs @@ -17,6 +17,7 @@ use zksync_types::{ fee::Fee, fee_model::BatchFeeInput, l2::L2Tx, + protocol_version::ProtocolSemanticVersion, snapshots::SnapshotRecoveryStatus, transaction_request::PaymasterParams, tx::{tx_execution_info::TxExecutionStatus, ExecutionMetrics, TransactionExecutionResult}, @@ -163,8 +164,8 @@ impl Snapshot { l1_batch: L1BatchNumber, l2_block: L2BlockNumber, storage_logs: &[StorageLog], + genesis_params: GenesisParams, ) -> Self { - let genesis_params = GenesisParams::mock(); let contracts = genesis_params.base_system_contracts(); let l1_batch = L1BatchHeader::new( l1_batch, @@ -208,7 +209,11 @@ pub async fn prepare_recovery_snapshot( l2_block: L2BlockNumber, storage_logs: &[StorageLog], ) -> SnapshotRecoveryStatus { - recover(storage, Snapshot::make(l1_batch, l2_block, storage_logs)).await + recover( + storage, + Snapshot::make(l1_batch, l2_block, storage_logs, GenesisParams::mock()), + ) + .await } /// Takes a storage snapshot at the last sealed L1 batch. @@ -290,6 +295,10 @@ pub async fn recover( .protocol_versions_dal() .save_protocol_version_with_tx(&ProtocolVersion { base_system_contracts_hashes: snapshot.l1_batch.base_system_contracts_hashes, + version: ProtocolSemanticVersion { + minor: snapshot.l1_batch.protocol_version.unwrap(), + patch: 0.into(), + }, ..ProtocolVersion::default() }) .await diff --git a/core/tests/test_account/Cargo.toml b/core/tests/test_account/Cargo.toml index 0b2e7aa9340f..6df10edd7dca 100644 --- a/core/tests/test_account/Cargo.toml +++ b/core/tests/test_account/Cargo.toml @@ -19,3 +19,4 @@ zksync_contracts.workspace = true hex.workspace = true ethabi.workspace = true +rand.workspace = true diff --git a/core/tests/test_account/src/lib.rs b/core/tests/test_account/src/lib.rs index 619caeb1ebd5..e259ce209c63 100644 --- a/core/tests/test_account/src/lib.rs +++ b/core/tests/test_account/src/lib.rs @@ -50,6 +50,10 @@ impl Account { Self::new(K256PrivateKey::random()) } + pub fn random_using(rng: &mut impl rand::Rng) -> Self { + Self::new(K256PrivateKey::random_using(rng)) + } + pub fn get_l2_tx_for_execute(&mut self, execute: Execute, fee: Option) -> Transaction { let tx = self.get_l2_tx_for_execute_with_nonce(execute, fee, self.nonce); self.nonce += 1; diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 44c2a8b8395f..7b30b67c2650 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -9536,6 +9536,7 @@ version = "0.1.0" dependencies = [ "ethabi", "hex", + "rand 0.8.5", "zksync_contracts", "zksync_eth_signer", "zksync_system_constants", From 7842bc4842c5c92437639105d8edac5f775ad0e6 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Wed, 19 Jun 2024 13:53:04 +0400 Subject: [PATCH 19/43] feat(node): Port (most of) Node to the Node Framework (#2196) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Ports most of the EN parts to the framework, except: - Reorg detector - Genesis / snapshot recovery - Unique metrics stuff ## Why ❔ This forms the "main" body of the framework-based EN. The binary can already run, it can proxy and re-execute transactions, etc. The remaining work may be somewhat complex, so I would like to ship it separately -- so that it receives thorough review. There are some TODOs without a task number in this PR, expect them either to be fixed in the next one, or otherwise an issue to be added. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- Cargo.lock | 2 + checks-config/era.dic | 1 + core/bin/external_node/Cargo.toml | 1 + core/bin/external_node/src/main.rs | 31 +- core/bin/external_node/src/node_builder.rs | 508 ++++++++++++++++++ core/bin/external_node/src/tests.rs | 2 + core/bin/zksync_server/src/node_builder.rs | 50 +- core/node/api_server/src/tx_sender/proxy.rs | 22 +- core/node/node_framework/Cargo.toml | 1 + .../node/node_framework/examples/main_node.rs | 27 +- .../layers/batch_status_updater.rs | 52 ++ .../layers/commitment_generator.rs | 24 +- .../src/implementations/layers/consensus.rs | 4 +- .../layers/consistency_checker.rs | 12 +- .../implementations/layers/house_keeper.rs | 32 -- .../l1_batch_commitment_mode_validation.rs | 59 ++ .../layers/main_node_client.rs | 19 +- .../layers/main_node_fee_params_fetcher.rs | 46 ++ .../layers/metadata_calculator.rs | 31 +- .../src/implementations/layers/mod.rs | 8 + .../layers/postgres_metrics.rs | 57 ++ .../layers/prometheus_exporter.rs | 8 +- .../src/implementations/layers/pruning.rs | 75 +++ .../layers/reorg_detector_checker.rs | 19 +- .../layers/state_keeper/external_io.rs | 68 +++ .../state_keeper/main_batch_executor.rs | 12 +- .../layers/state_keeper/mempool_io.rs | 51 +- .../layers/state_keeper/mod.rs | 29 +- .../layers/state_keeper/output_handler.rs | 121 +++++ .../layers/sync_state_updater.rs | 75 +++ .../layers/tree_data_fetcher.rs | 67 +++ .../layers/validate_chain_ids.rs | 61 +++ .../implementations/layers/web3_api/server.rs | 14 +- .../layers/web3_api/tx_sender.rs | 70 ++- .../layers/web3_api/tx_sink.rs | 31 +- .../node_sync/src/validate_chain_ids_task.rs | 17 + .../src/batch_executor/main_executor.rs | 14 +- etc/env/configs/ext-node.toml | 2 + 38 files changed, 1569 insertions(+), 154 deletions(-) create mode 100644 core/bin/external_node/src/node_builder.rs create mode 100644 core/node/node_framework/src/implementations/layers/batch_status_updater.rs create mode 100644 core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs create mode 100644 core/node/node_framework/src/implementations/layers/main_node_fee_params_fetcher.rs create mode 100644 core/node/node_framework/src/implementations/layers/postgres_metrics.rs create mode 100644 core/node/node_framework/src/implementations/layers/pruning.rs create mode 100644 core/node/node_framework/src/implementations/layers/state_keeper/external_io.rs create mode 100644 core/node/node_framework/src/implementations/layers/state_keeper/output_handler.rs create mode 100644 core/node/node_framework/src/implementations/layers/sync_state_updater.rs create mode 100644 core/node/node_framework/src/implementations/layers/tree_data_fetcher.rs create mode 100644 core/node/node_framework/src/implementations/layers/validate_chain_ids.rs diff --git a/Cargo.lock b/Cargo.lock index c41faf9d1faf..be0ffd1566b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8643,6 +8643,7 @@ dependencies = [ "zksync_node_consensus", "zksync_node_db_pruner", "zksync_node_fee_model", + "zksync_node_framework", "zksync_node_genesis", "zksync_node_sync", "zksync_object_store", @@ -8954,6 +8955,7 @@ dependencies = [ "zksync_metadata_calculator", "zksync_node_api_server", "zksync_node_consensus", + "zksync_node_db_pruner", "zksync_node_fee_model", "zksync_node_sync", "zksync_object_store", diff --git a/checks-config/era.dic b/checks-config/era.dic index a93a467f956e..0b55a55c83ea 100644 --- a/checks-config/era.dic +++ b/checks-config/era.dic @@ -962,6 +962,7 @@ zksync_merkle_tree TreeMetadata delegator decrement +whitelisted Bbellman Sbellman DCMAKE diff --git a/core/bin/external_node/Cargo.toml b/core/bin/external_node/Cargo.toml index d4a883b190f4..ee6aa08be9da 100644 --- a/core/bin/external_node/Cargo.toml +++ b/core/bin/external_node/Cargo.toml @@ -42,6 +42,7 @@ zksync_metadata_calculator.workspace = true zksync_node_sync.workspace = true zksync_node_api_server.workspace = true zksync_node_consensus.workspace = true +zksync_node_framework.workspace = true vlog.workspace = true zksync_concurrency.workspace = true diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index cca61889ff92..04435f66bf4b 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -3,6 +3,7 @@ use std::{collections::HashSet, net::Ipv4Addr, str::FromStr, sync::Arc, time::Du use anyhow::Context as _; use clap::Parser; use metrics::EN_METRICS; +use node_builder::ExternalNodeBuilder; use tokio::{ sync::{oneshot, watch, RwLock}, task::{self, JoinHandle}, @@ -63,6 +64,7 @@ mod config; mod init; mod metadata; mod metrics; +mod node_builder; #[cfg(test)] mod tests; @@ -426,10 +428,11 @@ async fn run_api( .build() .await .context("failed to build a proxy_cache_updater_pool")?; - task_handles.push(tokio::spawn(tx_proxy.run_account_nonce_sweeper( - proxy_cache_updater_pool.clone(), - stop_receiver.clone(), - ))); + task_handles.push(tokio::spawn( + tx_proxy + .account_nonce_sweeper_task(proxy_cache_updater_pool.clone()) + .run(stop_receiver.clone()), + )); let fee_params_fetcher_handle = tokio::spawn(fee_params_fetcher.clone().run(stop_receiver.clone())); @@ -701,6 +704,10 @@ struct Cli { /// Comma-separated list of components to launch. #[arg(long, default_value = "all")] components: ComponentsToRun, + + /// Run the node using the node framework. + #[arg(long)] + use_node_framework: bool, } #[derive(Debug, Clone, Copy, PartialEq, Hash, Eq)] @@ -784,6 +791,22 @@ async fn main() -> anyhow::Result<()> { .fetch_remote(main_node_client.as_ref()) .await .context("failed fetching remote part of node config from main node")?; + + // If the node framework is used, run the node. + if opt.use_node_framework { + // We run the node from a different thread, since the current thread is in tokio context. + std::thread::spawn(move || { + let node = + ExternalNodeBuilder::new(config).build(opt.components.0.into_iter().collect())?; + node.run()?; + anyhow::Ok(()) + }) + .join() + .expect("Failed to run the node")?; + + return Ok(()); + } + if let Some(threshold) = config.optional.slow_query_threshold() { ConnectionPool::::global_config().set_slow_query_threshold(threshold)?; } diff --git a/core/bin/external_node/src/node_builder.rs b/core/bin/external_node/src/node_builder.rs new file mode 100644 index 000000000000..5eaff63d20a0 --- /dev/null +++ b/core/bin/external_node/src/node_builder.rs @@ -0,0 +1,508 @@ +//! This module provides a "builder" for the external node, +//! as well as an interface to run the node with the specified components. + +use anyhow::Context as _; +use zksync_config::{ + configs::{ + api::{HealthCheckConfig, MerkleTreeApiConfig}, + database::MerkleTreeMode, + DatabaseSecrets, + }, + PostgresConfig, +}; +use zksync_metadata_calculator::{MetadataCalculatorConfig, MetadataCalculatorRecoveryConfig}; +use zksync_node_api_server::{tx_sender::ApiContracts, web3::Namespace}; +use zksync_node_framework::{ + implementations::layers::{ + batch_status_updater::BatchStatusUpdaterLayer, + commitment_generator::CommitmentGeneratorLayer, + consensus::{ConsensusLayer, Mode}, + consistency_checker::ConsistencyCheckerLayer, + healtcheck_server::HealthCheckLayer, + l1_batch_commitment_mode_validation::L1BatchCommitmentModeValidationLayer, + main_node_client::MainNodeClientLayer, + main_node_fee_params_fetcher::MainNodeFeeParamsFetcherLayer, + metadata_calculator::MetadataCalculatorLayer, + pools_layer::PoolsLayerBuilder, + postgres_metrics::PostgresMetricsLayer, + prometheus_exporter::PrometheusExporterLayer, + pruning::PruningLayer, + query_eth_client::QueryEthClientLayer, + sigint::SigintHandlerLayer, + state_keeper::{ + external_io::ExternalIOLayer, main_batch_executor::MainBatchExecutorLayer, + output_handler::OutputHandlerLayer, StateKeeperLayer, + }, + sync_state_updater::SyncStateUpdaterLayer, + tree_data_fetcher::TreeDataFetcherLayer, + validate_chain_ids::ValidateChainIdsLayer, + web3_api::{ + caches::MempoolCacheLayer, + server::{Web3ServerLayer, Web3ServerOptionalConfig}, + tree_api_client::TreeApiClientLayer, + tx_sender::{PostgresStorageCachesConfig, TxSenderLayer}, + tx_sink::TxSinkLayer, + }, + }, + service::{ZkStackService, ZkStackServiceBuilder}, +}; +use zksync_state::RocksdbStorageOptions; + +use crate::{ + config::{self, ExternalNodeConfig}, + Component, +}; + +/// Builder for the external node. +#[derive(Debug)] +pub(crate) struct ExternalNodeBuilder { + node: ZkStackServiceBuilder, + config: ExternalNodeConfig, +} + +impl ExternalNodeBuilder { + pub fn new(config: ExternalNodeConfig) -> Self { + Self { + node: ZkStackServiceBuilder::new(), + config, + } + } + + fn add_sigint_handler_layer(mut self) -> anyhow::Result { + self.node.add_layer(SigintHandlerLayer); + Ok(self) + } + + fn add_pools_layer(mut self) -> anyhow::Result { + // Note: the EN config doesn't currently support specifying configuration for replicas, + // so we reuse the master configuration for that purpose. + // Settings unconditionally set to `None` are either not supported by the EN configuration layer + // or are not used in the context of the external node. + let config = PostgresConfig { + max_connections: Some(self.config.postgres.max_connections), + max_connections_master: Some(self.config.postgres.max_connections), + acquire_timeout_sec: None, + statement_timeout_sec: None, + long_connection_threshold_ms: None, + slow_query_threshold_ms: self + .config + .optional + .slow_query_threshold() + .map(|d| d.as_millis() as u64), + test_server_url: None, + test_prover_url: None, + }; + let secrets = DatabaseSecrets { + server_url: Some(self.config.postgres.database_url()), + server_replica_url: Some(self.config.postgres.database_url()), + prover_url: None, + }; + let pools_layer = PoolsLayerBuilder::empty(config, secrets) + .with_master(true) + .with_replica(true) + .build(); + self.node.add_layer(pools_layer); + Ok(self) + } + + fn add_postgres_metrics_layer(mut self) -> anyhow::Result { + self.node.add_layer(PostgresMetricsLayer); + Ok(self) + } + + fn add_main_node_client_layer(mut self) -> anyhow::Result { + let layer = MainNodeClientLayer::new( + self.config.required.main_node_url.clone(), + self.config.optional.main_node_rate_limit_rps, + self.config.required.l2_chain_id, + ); + self.node.add_layer(layer); + Ok(self) + } + + fn add_healthcheck_layer(mut self) -> anyhow::Result { + let healthcheck_config = HealthCheckConfig { + port: self.config.required.healthcheck_port, + slow_time_limit_ms: self + .config + .optional + .healthcheck_slow_time_limit() + .map(|d| d.as_millis() as u64), + hard_time_limit_ms: self + .config + .optional + .healthcheck_hard_time_limit() + .map(|d| d.as_millis() as u64), + }; + self.node.add_layer(HealthCheckLayer(healthcheck_config)); + Ok(self) + } + + fn add_prometheus_exporter_layer(mut self) -> anyhow::Result { + if let Some(prom_config) = self.config.observability.prometheus() { + self.node.add_layer(PrometheusExporterLayer(prom_config)); + } else { + tracing::info!("No configuration for prometheus exporter, skipping"); + } + Ok(self) + } + + fn add_query_eth_client_layer(mut self) -> anyhow::Result { + let query_eth_client_layer = QueryEthClientLayer::new( + self.config.required.l1_chain_id, + self.config.required.eth_client_url.clone(), + ); + self.node.add_layer(query_eth_client_layer); + Ok(self) + } + + fn add_state_keeper_layer(mut self) -> anyhow::Result { + // While optional bytecode compression may be disabled on the main node, there are batches where + // optional bytecode compression was enabled. To process these batches (and also for the case where + // compression will become optional on the sequencer again), EN has to allow txs without bytecode + // compression. + const OPTIONAL_BYTECODE_COMPRESSION: bool = true; + + let persistence_layer = OutputHandlerLayer::new( + self.config + .remote + .l2_shared_bridge_addr + .expect("L2 shared bridge address is not set"), + self.config.optional.l2_block_seal_queue_capacity, + ) + .with_pre_insert_txs(true) // EN requires txs to be pre-inserted. + .with_protective_reads_persistence_enabled( + self.config.optional.protective_reads_persistence_enabled, + ); + + let io_layer = ExternalIOLayer::new(self.config.required.l2_chain_id); + + // We only need call traces on the external node if the `debug_` namespace is enabled. + let save_call_traces = self + .config + .optional + .api_namespaces() + .contains(&Namespace::Debug); + let main_node_batch_executor_builder_layer = + MainBatchExecutorLayer::new(save_call_traces, OPTIONAL_BYTECODE_COMPRESSION); + + let rocksdb_options = RocksdbStorageOptions { + block_cache_capacity: self + .config + .experimental + .state_keeper_db_block_cache_capacity(), + max_open_files: self.config.experimental.state_keeper_db_max_open_files, + }; + let state_keeper_layer = StateKeeperLayer::new( + self.config.required.state_cache_path.clone(), + rocksdb_options, + ); + self.node + .add_layer(persistence_layer) + .add_layer(io_layer) + .add_layer(main_node_batch_executor_builder_layer) + .add_layer(state_keeper_layer); + Ok(self) + } + + fn add_consensus_layer(mut self) -> anyhow::Result { + let config = self.config.consensus.clone(); + let secrets = + config::read_consensus_secrets().context("config::read_consensus_secrets()")?; + let layer = ConsensusLayer { + mode: Mode::External, + config, + secrets, + }; + self.node.add_layer(layer); + Ok(self) + } + + fn add_pruning_layer(mut self) -> anyhow::Result { + if self.config.optional.pruning_enabled { + let layer = PruningLayer::new( + self.config.optional.pruning_removal_delay(), + self.config.optional.pruning_chunk_size, + self.config.optional.pruning_data_retention(), + ); + self.node.add_layer(layer); + } else { + tracing::info!("Pruning is disabled"); + } + Ok(self) + } + + fn add_l1_batch_commitment_mode_validation_layer(mut self) -> anyhow::Result { + let layer = L1BatchCommitmentModeValidationLayer::new( + self.config.remote.diamond_proxy_addr, + self.config.optional.l1_batch_commit_data_generator_mode, + ); + self.node.add_layer(layer); + Ok(self) + } + + fn add_validate_chain_ids_layer(mut self) -> anyhow::Result { + let layer = ValidateChainIdsLayer::new( + self.config.required.l1_chain_id, + self.config.required.l2_chain_id, + ); + self.node.add_layer(layer); + Ok(self) + } + + fn add_consistency_checker_layer(mut self) -> anyhow::Result { + let max_batches_to_recheck = 10; // TODO (BFT-97): Make it a part of a proper EN config + let layer = ConsistencyCheckerLayer::new( + self.config.remote.diamond_proxy_addr, + max_batches_to_recheck, + self.config.optional.l1_batch_commit_data_generator_mode, + ); + self.node.add_layer(layer); + Ok(self) + } + + fn add_commitment_generator_layer(mut self) -> anyhow::Result { + let layer = + CommitmentGeneratorLayer::new(self.config.optional.l1_batch_commit_data_generator_mode) + .with_max_parallelism( + self.config + .experimental + .commitment_generator_max_parallelism, + ); + self.node.add_layer(layer); + Ok(self) + } + + fn add_batch_status_updater_layer(mut self) -> anyhow::Result { + let layer = BatchStatusUpdaterLayer; + self.node.add_layer(layer); + Ok(self) + } + + fn add_tree_data_fetcher_layer(mut self) -> anyhow::Result { + let layer = TreeDataFetcherLayer::new(self.config.remote.diamond_proxy_addr); + self.node.add_layer(layer); + Ok(self) + } + + fn add_sync_state_updater_layer(mut self) -> anyhow::Result { + // This layer may be used as a fallback for EN API if API server runs without the core component. + self.node.add_layer(SyncStateUpdaterLayer); + Ok(self) + } + + fn add_metadata_calculator_layer(mut self, with_tree_api: bool) -> anyhow::Result { + let metadata_calculator_config = MetadataCalculatorConfig { + db_path: self.config.required.merkle_tree_path.clone(), + max_open_files: self.config.optional.merkle_tree_max_open_files, + mode: MerkleTreeMode::Lightweight, + delay_interval: self.config.optional.merkle_tree_processing_delay(), + max_l1_batches_per_iter: self.config.optional.merkle_tree_max_l1_batches_per_iter, + multi_get_chunk_size: self.config.optional.merkle_tree_multi_get_chunk_size, + block_cache_capacity: self.config.optional.merkle_tree_block_cache_size(), + include_indices_and_filters_in_block_cache: self + .config + .optional + .merkle_tree_include_indices_and_filters_in_block_cache, + memtable_capacity: self.config.optional.merkle_tree_memtable_capacity(), + stalled_writes_timeout: self.config.optional.merkle_tree_stalled_writes_timeout(), + recovery: MetadataCalculatorRecoveryConfig { + desired_chunk_size: self.config.experimental.snapshots_recovery_tree_chunk_size, + parallel_persistence_buffer: self + .config + .experimental + .snapshots_recovery_tree_parallel_persistence_buffer, + }, + }; + + // Configure basic tree layer. + let mut layer = MetadataCalculatorLayer::new(metadata_calculator_config); + + // Add tree API if needed. + if with_tree_api { + let merkle_tree_api_config = MerkleTreeApiConfig { + port: self + .config + .tree_component + .api_port + .context("should contain tree api port")?, + }; + layer = layer.with_tree_api_config(merkle_tree_api_config); + } + + // Add tree pruning if needed. + if self.config.optional.pruning_enabled { + layer = layer.with_pruning_config(self.config.optional.pruning_removal_delay()); + } + + self.node.add_layer(layer); + Ok(self) + } + + fn add_tx_sender_layer(mut self) -> anyhow::Result { + let postgres_storage_config = PostgresStorageCachesConfig { + factory_deps_cache_size: self.config.optional.factory_deps_cache_size() as u64, + initial_writes_cache_size: self.config.optional.initial_writes_cache_size() as u64, + latest_values_cache_size: self.config.optional.latest_values_cache_size() as u64, + }; + let max_vm_concurrency = self.config.optional.vm_concurrency_limit; + let api_contracts = ApiContracts::load_from_disk_blocking(); // TODO (BFT-138): Allow to dynamically reload API contracts; + let tx_sender_layer = TxSenderLayer::new( + (&self.config).into(), + postgres_storage_config, + max_vm_concurrency, + api_contracts, + ) + .with_whitelisted_tokens_for_aa_cache(true); + + self.node.add_layer(TxSinkLayer::ProxySink); + self.node.add_layer(tx_sender_layer); + Ok(self) + } + + fn add_mempool_cache_layer(mut self) -> anyhow::Result { + self.node.add_layer(MempoolCacheLayer::new( + self.config.optional.mempool_cache_size, + self.config.optional.mempool_cache_update_interval(), + )); + Ok(self) + } + + fn add_tree_api_client_layer(mut self) -> anyhow::Result { + self.node.add_layer(TreeApiClientLayer::http( + self.config.api_component.tree_api_remote_url.clone(), + )); + Ok(self) + } + + fn add_main_node_fee_params_fetcher_layer(mut self) -> anyhow::Result { + self.node.add_layer(MainNodeFeeParamsFetcherLayer); + Ok(self) + } + + fn web3_api_optional_config(&self) -> Web3ServerOptionalConfig { + // The refresh interval should be several times lower than the pruning removal delay, so that + // soft-pruning will timely propagate to the API server. + let pruning_info_refresh_interval = self.config.optional.pruning_removal_delay() / 5; + + Web3ServerOptionalConfig { + namespaces: Some(self.config.optional.api_namespaces()), + filters_limit: Some(self.config.optional.filters_limit), + subscriptions_limit: Some(self.config.optional.filters_limit), + batch_request_size_limit: Some(self.config.optional.max_batch_request_size), + response_body_size_limit: Some(self.config.optional.max_response_body_size()), + with_extended_tracing: self.config.optional.extended_rpc_tracing, + pruning_info_refresh_interval: Some(pruning_info_refresh_interval), + websocket_requests_per_minute_limit: None, // To be set by WS server layer method if required. + replication_lag_limit: None, // TODO: Support replication lag limit + } + } + + fn add_http_web3_api_layer(mut self) -> anyhow::Result { + let optional_config = self.web3_api_optional_config(); + self.node.add_layer(Web3ServerLayer::http( + self.config.required.http_port, + (&self.config).into(), + optional_config, + )); + + Ok(self) + } + + fn add_ws_web3_api_layer(mut self) -> anyhow::Result { + // TODO: Support websocket requests per minute limit + let optional_config = self.web3_api_optional_config(); + self.node.add_layer(Web3ServerLayer::ws( + self.config.required.ws_port, + (&self.config).into(), + optional_config, + )); + + Ok(self) + } + + pub fn build(mut self, mut components: Vec) -> anyhow::Result { + // Add "base" layers + self = self + .add_sigint_handler_layer()? + .add_healthcheck_layer()? + .add_prometheus_exporter_layer()? + .add_pools_layer()? + .add_main_node_client_layer()? + .add_query_eth_client_layer()?; + + // Add preconditions for all the components. + self = self + .add_l1_batch_commitment_mode_validation_layer()? + .add_validate_chain_ids_layer()?; + + // Sort the components, so that the components they may depend on each other are added in the correct order. + components.sort_unstable_by_key(|component| match component { + // API consumes the resources provided by other layers (multiple ones), so it has to come the last. + Component::HttpApi | Component::WsApi => 1, + // Default priority. + _ => 0, + }); + + for component in &components { + match component { + Component::HttpApi => { + self = self + .add_sync_state_updater_layer()? + .add_mempool_cache_layer()? + .add_tree_api_client_layer()? + .add_main_node_fee_params_fetcher_layer()? + .add_tx_sender_layer()? + .add_http_web3_api_layer()?; + } + Component::WsApi => { + self = self + .add_sync_state_updater_layer()? + .add_mempool_cache_layer()? + .add_tree_api_client_layer()? + .add_main_node_fee_params_fetcher_layer()? + .add_tx_sender_layer()? + .add_ws_web3_api_layer()?; + } + Component::Tree => { + // Right now, distributed mode for EN is not fully supported, e.g. there are some + // issues with reorg detection and snapshot recovery. + // So we require the core component to be present, e.g. forcing the EN to run in a monolithic mode. + anyhow::ensure!( + components.contains(&Component::Core), + "Tree must run on the same machine as Core" + ); + let with_tree_api = components.contains(&Component::TreeApi); + self = self.add_metadata_calculator_layer(with_tree_api)?; + } + Component::TreeApi => { + anyhow::ensure!( + components.contains(&Component::Tree), + "Merkle tree API cannot be started without a tree component" + ); + // Do nothing, will be handled by the `Tree` component. + } + Component::TreeFetcher => { + self = self.add_tree_data_fetcher_layer()?; + } + Component::Core => { + // Core is a singleton & mandatory component, + // so until we have a dedicated component for "auxiliary" tasks, + // it's responsible for things like metrics. + self = self.add_postgres_metrics_layer()?; + + // Main tasks + self = self + .add_state_keeper_layer()? + .add_consensus_layer()? + .add_pruning_layer()? + .add_consistency_checker_layer()? + .add_commitment_generator_layer()? + .add_batch_status_updater_layer()?; + } + } + } + + Ok(self.node.build()?) + } +} diff --git a/core/bin/external_node/src/tests.rs b/core/bin/external_node/src/tests.rs index c78c5329386e..8966a7ac3f3b 100644 --- a/core/bin/external_node/src/tests.rs +++ b/core/bin/external_node/src/tests.rs @@ -157,6 +157,7 @@ async fn external_node_basics(components_str: &'static str) { let opt = Cli { enable_consensus: false, components, + use_node_framework: false, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { @@ -265,6 +266,7 @@ async fn node_reacts_to_stop_signal_during_initial_reorg_detection() { let opt = Cli { enable_consensus: false, components: "core".parse().unwrap(), + use_node_framework: false, }; let mut config = ExternalNodeConfig::mock(&temp_dir, &connection_pool); if opt.components.0.contains(&Component::TreeApi) { diff --git a/core/bin/zksync_server/src/node_builder.rs b/core/bin/zksync_server/src/node_builder.rs index 551683605479..096d5e783551 100644 --- a/core/bin/zksync_server/src/node_builder.rs +++ b/core/bin/zksync_server/src/node_builder.rs @@ -23,18 +23,20 @@ use zksync_node_framework::{ eth_watch::EthWatchLayer, healtcheck_server::HealthCheckLayer, house_keeper::HouseKeeperLayer, + l1_batch_commitment_mode_validation::L1BatchCommitmentModeValidationLayer, l1_gas::SequencerL1GasLayer, metadata_calculator::MetadataCalculatorLayer, object_store::ObjectStoreLayer, pk_signing_eth_client::PKSigningEthClientLayer, pools_layer::PoolsLayerBuilder, + postgres_metrics::PostgresMetricsLayer, prometheus_exporter::PrometheusExporterLayer, proof_data_handler::ProofDataHandlerLayer, query_eth_client::QueryEthClientLayer, sigint::SigintHandlerLayer, state_keeper::{ main_batch_executor::MainBatchExecutorLayer, mempool_io::MempoolIOLayer, - StateKeeperLayer, + output_handler::OutputHandlerLayer, RocksdbStorageOptions, StateKeeperLayer, }, tee_verifier_input_producer::TeeVerifierInputProducerLayer, vm_runner::protective_reads::ProtectiveReadsWriterLayer, @@ -111,6 +113,11 @@ impl MainNodeBuilder { Ok(self) } + fn add_postgres_metrics_layer(mut self) -> anyhow::Result { + self.node.add_layer(PostgresMetricsLayer); + Ok(self) + } + fn add_pk_signing_client_layer(mut self) -> anyhow::Result { let eth_config = try_load_config!(self.configs.eth); let wallets = try_load_config!(self.wallets.eth_sender); @@ -155,6 +162,15 @@ impl MainNodeBuilder { Ok(self) } + fn add_l1_batch_commitment_mode_validation_layer(mut self) -> anyhow::Result { + let layer = L1BatchCommitmentModeValidationLayer::new( + self.contracts_config.diamond_proxy_addr, + self.genesis_config.l1_batch_commit_data_generator_mode, + ); + self.node.add_layer(layer); + Ok(self) + } + fn add_metadata_calculator_layer(mut self, with_tree_api: bool) -> anyhow::Result { let merkle_tree_env_config = try_load_config!(self.configs.db_config).merkle_tree; let operations_manager_env_config = @@ -173,19 +189,37 @@ impl MainNodeBuilder { } fn add_state_keeper_layer(mut self) -> anyhow::Result { + // Bytecode compression is currently mandatory for the transactions processed by the sequencer. + const OPTIONAL_BYTECODE_COMPRESSION: bool = false; + let wallets = self.wallets.clone(); let sk_config = try_load_config!(self.configs.state_keeper_config); + let persistence_layer = OutputHandlerLayer::new( + self.contracts_config + .l2_shared_bridge_addr + .context("L2 shared bridge address")?, + sk_config.l2_block_seal_queue_capacity, + ); let mempool_io_layer = MempoolIOLayer::new( self.genesis_config.l2_chain_id, - self.contracts_config.clone(), sk_config.clone(), try_load_config!(self.configs.mempool_config), try_load_config!(wallets.state_keeper), ); let db_config = try_load_config!(self.configs.db_config); - let main_node_batch_executor_builder_layer = MainBatchExecutorLayer::new(sk_config); - let state_keeper_layer = StateKeeperLayer::new(db_config); + let main_node_batch_executor_builder_layer = + MainBatchExecutorLayer::new(sk_config.save_call_traces, OPTIONAL_BYTECODE_COMPRESSION); + + let rocksdb_options = RocksdbStorageOptions { + block_cache_capacity: db_config + .experimental + .state_keeper_db_block_cache_capacity(), + max_open_files: db_config.experimental.state_keeper_db_max_open_files, + }; + let state_keeper_layer = + StateKeeperLayer::new(db_config.state_keeper_db_path, rocksdb_options); self.node + .add_layer(persistence_layer) .add_layer(mempool_io_layer) .add_layer(main_node_batch_executor_builder_layer) .add_layer(state_keeper_layer); @@ -308,6 +342,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + ..Default::default() }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, @@ -419,7 +454,8 @@ impl MainNodeBuilder { .add_healthcheck_layer()? .add_prometheus_exporter_layer()? .add_query_eth_client_layer()? - .add_sequencer_l1_gas_layer()?; + .add_sequencer_l1_gas_layer()? + .add_l1_batch_commitment_mode_validation_layer()?; // Sort the components, so that the components they may depend on each other are added in the correct order. components.sort_unstable_by_key(|component| match component { @@ -479,7 +515,9 @@ impl MainNodeBuilder { self = self.add_tee_verifier_input_producer_layer()?; } Component::Housekeeper => { - self = self.add_house_keeper_layer()?; + self = self + .add_house_keeper_layer()? + .add_postgres_metrics_layer()?; } Component::ProofDataHandler => { self = self.add_proof_data_handler_layer()?; diff --git a/core/node/api_server/src/tx_sender/proxy.rs b/core/node/api_server/src/tx_sender/proxy.rs index a1fa77d2f1b2..52fcc8a1a8b0 100644 --- a/core/node/api_server/src/tx_sender/proxy.rs +++ b/core/node/api_server/src/tx_sender/proxy.rs @@ -1,6 +1,5 @@ use std::{ collections::{BTreeSet, HashMap, HashSet}, - future::Future, sync::Arc, time::Duration, }; @@ -282,13 +281,24 @@ impl TxProxy { pending_nonce } - pub fn run_account_nonce_sweeper( + pub fn account_nonce_sweeper_task( &self, pool: ConnectionPool, - stop_receiver: watch::Receiver, - ) -> impl Future> { - let tx_cache = self.tx_cache.clone(); - tx_cache.run_updates(pool, stop_receiver) + ) -> AccountNonceSweeperTask { + let cache = self.tx_cache.clone(); + AccountNonceSweeperTask { cache, pool } + } +} + +#[derive(Debug)] +pub struct AccountNonceSweeperTask { + cache: TxCache, + pool: ConnectionPool, +} + +impl AccountNonceSweeperTask { + pub async fn run(self, stop_receiver: watch::Receiver) -> anyhow::Result<()> { + self.cache.run_updates(self.pool, stop_receiver).await } } diff --git a/core/node/node_framework/Cargo.toml b/core/node/node_framework/Cargo.toml index 8e2c915d5749..d48522fb8116 100644 --- a/core/node/node_framework/Cargo.toml +++ b/core/node/node_framework/Cargo.toml @@ -45,6 +45,7 @@ zksync_tee_verifier_input_producer.workspace = true zksync_queued_job_processor.workspace = true zksync_reorg_detector.workspace = true zksync_vm_runner.workspace = true +zksync_node_db_pruner.workspace = true tracing.workspace = true thiserror.workspace = true diff --git a/core/node/node_framework/examples/main_node.rs b/core/node/node_framework/examples/main_node.rs index a62f04af0334..f0cb8417ff97 100644 --- a/core/node/node_framework/examples/main_node.rs +++ b/core/node/node_framework/examples/main_node.rs @@ -43,7 +43,7 @@ use zksync_node_framework::{ sigint::SigintHandlerLayer, state_keeper::{ main_batch_executor::MainBatchExecutorLayer, mempool_io::MempoolIOLayer, - StateKeeperLayer, + output_handler::OutputHandlerLayer, StateKeeperLayer, }, web3_api::{ caches::MempoolCacheLayer, @@ -55,6 +55,7 @@ use zksync_node_framework::{ }, service::{ZkStackService, ZkStackServiceBuilder, ZkStackServiceError}, }; +use zksync_state::RocksdbStorageOptions; struct MainNodeBuilder { node: ZkStackServiceBuilder, @@ -145,17 +146,32 @@ impl MainNodeBuilder { fn add_state_keeper_layer(mut self) -> anyhow::Result { let wallets = Wallets::from_env()?; + let contracts_config = ContractsConfig::from_env()?; + let sk_config = StateKeeperConfig::from_env()?; + let persisence_layer = OutputHandlerLayer::new( + contracts_config.l2_shared_bridge_addr.unwrap(), + sk_config.l2_block_seal_queue_capacity, + ); let mempool_io_layer = MempoolIOLayer::new( NetworkConfig::from_env()?.zksync_network_id, - ContractsConfig::from_env()?, - StateKeeperConfig::from_env()?, + sk_config, MempoolConfig::from_env()?, wallets.state_keeper.context("State keeper wallets")?, ); let main_node_batch_executor_builder_layer = - MainBatchExecutorLayer::new(StateKeeperConfig::from_env()?); - let state_keeper_layer = StateKeeperLayer::new(DBConfig::from_env()?); + MainBatchExecutorLayer::new(StateKeeperConfig::from_env()?.save_call_traces, true); + let db_config = DBConfig::from_env()?; + + let rocksdb_options = RocksdbStorageOptions { + block_cache_capacity: db_config + .experimental + .state_keeper_db_block_cache_capacity(), + max_open_files: db_config.experimental.state_keeper_db_max_open_files, + }; + let state_keeper_layer = + StateKeeperLayer::new(db_config.state_keeper_db_path, rocksdb_options); self.node + .add_layer(persisence_layer) .add_layer(mempool_io_layer) .add_layer(main_node_batch_executor_builder_layer) .add_layer(state_keeper_layer); @@ -286,6 +302,7 @@ impl MainNodeBuilder { rpc_config.websocket_requests_per_minute_limit(), ), replication_lag_limit: circuit_breaker_config.replication_lag_limit(), + ..Default::default() }; self.node.add_layer(Web3ServerLayer::ws( rpc_config.ws_port, diff --git a/core/node/node_framework/src/implementations/layers/batch_status_updater.rs b/core/node/node_framework/src/implementations/layers/batch_status_updater.rs new file mode 100644 index 000000000000..ba328facc8a3 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/batch_status_updater.rs @@ -0,0 +1,52 @@ +use zksync_node_sync::batch_status_updater::BatchStatusUpdater; + +use crate::{ + implementations::resources::{ + healthcheck::AppHealthCheckResource, + main_node_client::MainNodeClientResource, + pools::{MasterPool, PoolResource}, + }, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct BatchStatusUpdaterLayer; + +#[async_trait::async_trait] +impl WiringLayer for BatchStatusUpdaterLayer { + fn layer_name(&self) -> &'static str { + "batch_status_updater_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let pool = context.get_resource::>().await?; + let MainNodeClientResource(client) = context.get_resource().await?; + + let updater = BatchStatusUpdater::new(client, pool.get().await?); + + // Insert healthcheck + let AppHealthCheckResource(app_health) = context.get_resource_or_default().await; + app_health + .insert_component(updater.health_check()) + .map_err(WiringError::internal)?; + + // Insert task + context.add_task(Box::new(updater)); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Task for BatchStatusUpdater { + fn id(&self) -> TaskId { + "batch_status_updater".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run(stop_receiver.0).await?; + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/commitment_generator.rs b/core/node/node_framework/src/implementations/layers/commitment_generator.rs index 5d2f63931295..cc57599759eb 100644 --- a/core/node/node_framework/src/implementations/layers/commitment_generator.rs +++ b/core/node/node_framework/src/implementations/layers/commitment_generator.rs @@ -1,3 +1,5 @@ +use std::num::NonZero; + use zksync_commitment_generator::CommitmentGenerator; use zksync_types::commitment::L1BatchCommitmentMode; @@ -14,11 +16,20 @@ use crate::{ #[derive(Debug)] pub struct CommitmentGeneratorLayer { mode: L1BatchCommitmentMode, + max_parallelism: Option>, } impl CommitmentGeneratorLayer { pub fn new(mode: L1BatchCommitmentMode) -> Self { - Self { mode } + Self { + mode, + max_parallelism: None, + } + } + + pub fn with_max_parallelism(mut self, max_parallelism: Option>) -> Self { + self.max_parallelism = max_parallelism; + self } } @@ -30,10 +41,17 @@ impl WiringLayer for CommitmentGeneratorLayer { async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { let pool_resource = context.get_resource::>().await?; - let pool_size = CommitmentGenerator::default_parallelism().get(); + + let pool_size = self + .max_parallelism + .unwrap_or(CommitmentGenerator::default_parallelism()) + .get(); let main_pool = pool_resource.get_custom(pool_size).await?; - let commitment_generator = CommitmentGenerator::new(main_pool, self.mode); + let mut commitment_generator = CommitmentGenerator::new(main_pool, self.mode); + if let Some(max_parallelism) = self.max_parallelism { + commitment_generator.set_max_parallelism(max_parallelism); + } let AppHealthCheckResource(app_health) = context.get_resource_or_default().await; app_health diff --git a/core/node/node_framework/src/implementations/layers/consensus.rs b/core/node/node_framework/src/implementations/layers/consensus.rs index 06bca1bba3ae..8cc7ea4098de 100644 --- a/core/node/node_framework/src/implementations/layers/consensus.rs +++ b/core/node/node_framework/src/implementations/layers/consensus.rs @@ -161,14 +161,14 @@ impl Task for FetcherTask { let root_ctx = ctx::root(); scope::run!(&root_ctx, |ctx, s| async { s.spawn_bg(consensus::era::run_en( - &root_ctx, + ctx, self.config, self.pool, self.sync_state, self.main_node_client, self.action_queue_sender, )); - ctx.wait(stop_receiver.0.wait_for(|stop| *stop)).await??; + let _ = stop_receiver.0.wait_for(|stop| *stop).await?; Ok(()) }) .await diff --git a/core/node/node_framework/src/implementations/layers/consistency_checker.rs b/core/node/node_framework/src/implementations/layers/consistency_checker.rs index a387fc19ead1..fb4b6d8f5eed 100644 --- a/core/node/node_framework/src/implementations/layers/consistency_checker.rs +++ b/core/node/node_framework/src/implementations/layers/consistency_checker.rs @@ -61,25 +61,19 @@ impl WiringLayer for ConsistencyCheckerLayer { .map_err(WiringError::internal)?; // Create and add tasks. - context.add_task(Box::new(ConsistencyCheckerTask { - consistency_checker, - })); + context.add_task(Box::new(consistency_checker)); Ok(()) } } -pub struct ConsistencyCheckerTask { - consistency_checker: ConsistencyChecker, -} - #[async_trait::async_trait] -impl Task for ConsistencyCheckerTask { +impl Task for ConsistencyChecker { fn id(&self) -> TaskId { "consistency_checker".into() } async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { - self.consistency_checker.run(stop_receiver.0).await + (*self).run(stop_receiver.0).await } } diff --git a/core/node/node_framework/src/implementations/layers/house_keeper.rs b/core/node/node_framework/src/implementations/layers/house_keeper.rs index 7b3e52c7ed5d..416d80691a31 100644 --- a/core/node/node_framework/src/implementations/layers/house_keeper.rs +++ b/core/node/node_framework/src/implementations/layers/house_keeper.rs @@ -1,10 +1,7 @@ -use std::time::Duration; - use zksync_config::configs::{ fri_prover_group::FriProverGroupConfig, house_keeper::HouseKeeperConfig, FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, }; -use zksync_dal::{metrics::PostgresMetrics, ConnectionPool, Core}; use zksync_house_keeper::{ blocks_state_reporter::L1BatchMetricsReporter, periodic_job::PeriodicJob, @@ -23,8 +20,6 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; -const SCRAPE_INTERVAL: Duration = Duration::from_secs(60); - #[derive(Debug)] pub struct HouseKeeperLayer { house_keeper_config: HouseKeeperConfig, @@ -67,9 +62,6 @@ impl WiringLayer for HouseKeeperLayer { let prover_pool = prover_pool_resource.get().await?; // initialize and add tasks - let pool_for_metrics = replica_pool_resource.get_singleton().await?; - context.add_task(Box::new(PostgresMetricsScrapingTask { pool_for_metrics })); - let l1_batch_metrics_reporter = L1BatchMetricsReporter::new( self.house_keeper_config .l1_batch_metrics_reporting_interval_ms, @@ -172,30 +164,6 @@ impl WiringLayer for HouseKeeperLayer { } } -#[derive(Debug)] -struct PostgresMetricsScrapingTask { - pool_for_metrics: ConnectionPool, -} - -#[async_trait::async_trait] -impl Task for PostgresMetricsScrapingTask { - fn id(&self) -> TaskId { - "postgres_metrics_scraping".into() - } - - async fn run(self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { - tokio::select! { - () = PostgresMetrics::run_scraping(self.pool_for_metrics, SCRAPE_INTERVAL) => { - tracing::warn!("Postgres metrics scraping unexpectedly stopped"); - } - _ = stop_receiver.0.changed() => { - tracing::info!("Stop signal received, Postgres metrics scraping is shutting down"); - } - } - Ok(()) - } -} - #[derive(Debug)] struct L1BatchMetricsReporterTask { l1_batch_metrics_reporter: L1BatchMetricsReporter, diff --git a/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs b/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs new file mode 100644 index 000000000000..e333eda51192 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/l1_batch_commitment_mode_validation.rs @@ -0,0 +1,59 @@ +use zksync_commitment_generator::validation_task::L1BatchCommitmentModeValidationTask; +use zksync_types::{commitment::L1BatchCommitmentMode, Address}; + +use crate::{ + implementations::resources::eth_interface::EthInterfaceResource, + precondition::Precondition, + service::{ServiceContext, StopReceiver}, + task::TaskId, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct L1BatchCommitmentModeValidationLayer { + diamond_proxy_addr: Address, + l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, +} + +impl L1BatchCommitmentModeValidationLayer { + pub fn new( + diamond_proxy_addr: Address, + l1_batch_commit_data_generator_mode: L1BatchCommitmentMode, + ) -> Self { + Self { + diamond_proxy_addr, + l1_batch_commit_data_generator_mode, + } + } +} + +#[async_trait::async_trait] +impl WiringLayer for L1BatchCommitmentModeValidationLayer { + fn layer_name(&self) -> &'static str { + "l1_batch_commitment_mode_validation_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let EthInterfaceResource(query_client) = context.get_resource().await?; + let task = L1BatchCommitmentModeValidationTask::new( + self.diamond_proxy_addr, + self.l1_batch_commit_data_generator_mode, + query_client, + ); + + context.add_precondition(Box::new(task)); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Precondition for L1BatchCommitmentModeValidationTask { + fn id(&self) -> TaskId { + "l1_batch_commitment_mode_validation".into() + } + + async fn check(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).exit_on_success().run(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/main_node_client.rs b/core/node/node_framework/src/implementations/layers/main_node_client.rs index 80e5d44c350f..a694eb831330 100644 --- a/core/node/node_framework/src/implementations/layers/main_node_client.rs +++ b/core/node/node_framework/src/implementations/layers/main_node_client.rs @@ -1,11 +1,14 @@ -use std::num::NonZeroUsize; +use std::{num::NonZeroUsize, sync::Arc}; use anyhow::Context; +use zksync_node_sync::MainNodeHealthCheck; use zksync_types::{url::SensitiveUrl, L2ChainId}; use zksync_web3_decl::client::{Client, DynClient, L2}; use crate::{ - implementations::resources::main_node_client::MainNodeClientResource, + implementations::resources::{ + healthcheck::AppHealthCheckResource, main_node_client::MainNodeClientResource, + }, service::ServiceContext, wiring_layer::{WiringError, WiringLayer}, }; @@ -40,9 +43,15 @@ impl WiringLayer for MainNodeClientLayer { .with_allowed_requests_per_second(self.rate_limit_rps) .build(); - context.insert_resource(MainNodeClientResource( - Box::new(main_node_client) as Box> - ))?; + let client = Box::new(main_node_client) as Box>; + context.insert_resource(MainNodeClientResource(client.clone()))?; + + // Insert healthcheck + let AppHealthCheckResource(app_health) = context.get_resource_or_default().await; + app_health + .insert_custom_component(Arc::new(MainNodeHealthCheck::from(client))) + .map_err(WiringError::internal)?; + Ok(()) } } diff --git a/core/node/node_framework/src/implementations/layers/main_node_fee_params_fetcher.rs b/core/node/node_framework/src/implementations/layers/main_node_fee_params_fetcher.rs new file mode 100644 index 000000000000..11bfab18a4c6 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/main_node_fee_params_fetcher.rs @@ -0,0 +1,46 @@ +use std::sync::Arc; + +use zksync_node_fee_model::l1_gas_price::MainNodeFeeParamsFetcher; + +use crate::{ + implementations::resources::{ + fee_input::FeeInputResource, main_node_client::MainNodeClientResource, + }, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct MainNodeFeeParamsFetcherLayer; + +#[async_trait::async_trait] +impl WiringLayer for MainNodeFeeParamsFetcherLayer { + fn layer_name(&self) -> &'static str { + "main_node_fee_params_fetcher_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let MainNodeClientResource(main_node_client) = context.get_resource().await?; + let fetcher = Arc::new(MainNodeFeeParamsFetcher::new(main_node_client)); + context.insert_resource(FeeInputResource(fetcher.clone()))?; + context.add_task(Box::new(MainNodeFeeParamsFetcherTask { fetcher })); + Ok(()) + } +} + +#[derive(Debug)] +struct MainNodeFeeParamsFetcherTask { + fetcher: Arc, +} + +#[async_trait::async_trait] +impl Task for MainNodeFeeParamsFetcherTask { + fn id(&self) -> TaskId { + "main_node_fee_params_fetcher".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + self.fetcher.run(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/metadata_calculator.rs b/core/node/node_framework/src/implementations/layers/metadata_calculator.rs index 935bb283fe81..bc1244410bf2 100644 --- a/core/node/node_framework/src/implementations/layers/metadata_calculator.rs +++ b/core/node/node_framework/src/implementations/layers/metadata_calculator.rs @@ -1,12 +1,13 @@ use std::{ net::{Ipv4Addr, SocketAddr}, sync::Arc, + time::Duration, }; use anyhow::Context as _; use zksync_config::configs::{api::MerkleTreeApiConfig, database::MerkleTreeMode}; use zksync_metadata_calculator::{ - LazyAsyncTreeReader, MetadataCalculator, MetadataCalculatorConfig, + LazyAsyncTreeReader, MerkleTreePruningTask, MetadataCalculator, MetadataCalculatorConfig, }; use zksync_storage::RocksDB; @@ -35,6 +36,7 @@ use crate::{ pub struct MetadataCalculatorLayer { config: MetadataCalculatorConfig, tree_api_config: Option, + pruning_config: Option, } impl MetadataCalculatorLayer { @@ -42,6 +44,7 @@ impl MetadataCalculatorLayer { Self { config, tree_api_config: None, + pruning_config: None, } } @@ -49,6 +52,11 @@ impl MetadataCalculatorLayer { self.tree_api_config = Some(tree_api_config); self } + + pub fn with_pruning_config(mut self, pruning_config: Duration) -> Self { + self.pruning_config = Some(pruning_config); + self + } } #[async_trait::async_trait] @@ -76,7 +84,7 @@ impl WiringLayer for MetadataCalculatorLayer { } }; - let metadata_calculator = MetadataCalculator::new( + let mut metadata_calculator = MetadataCalculator::new( self.config, object_store.map(|store_resource| store_resource.0), main_pool, @@ -98,6 +106,14 @@ impl WiringLayer for MetadataCalculatorLayer { })); } + if let Some(pruning_removal_delay) = self.pruning_config { + let pruning_task = Box::new(metadata_calculator.pruning_task(pruning_removal_delay)); + app_health + .insert_component(pruning_task.health_check()) + .map_err(|err| WiringError::Internal(err.into()))?; + context.add_task(pruning_task); + } + context.insert_resource(TreeApiClientResource(Arc::new( metadata_calculator.tree_reader(), )))?; @@ -154,3 +170,14 @@ impl Task for TreeApiTask { .await } } + +#[async_trait::async_trait] +impl Task for MerkleTreePruningTask { + fn id(&self) -> TaskId { + "merkle_tree_pruning_task".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/mod.rs b/core/node/node_framework/src/implementations/layers/mod.rs index 1c171e84b5ba..8637f15459d5 100644 --- a/core/node/node_framework/src/implementations/layers/mod.rs +++ b/core/node/node_framework/src/implementations/layers/mod.rs @@ -1,3 +1,4 @@ +pub mod batch_status_updater; pub mod circuit_breaker_checker; pub mod commitment_generator; pub mod consensus; @@ -7,19 +8,26 @@ pub mod eth_sender; pub mod eth_watch; pub mod healtcheck_server; pub mod house_keeper; +pub mod l1_batch_commitment_mode_validation; pub mod l1_gas; pub mod main_node_client; +pub mod main_node_fee_params_fetcher; pub mod metadata_calculator; pub mod object_store; pub mod pk_signing_eth_client; pub mod pools_layer; +pub mod postgres_metrics; pub mod prometheus_exporter; pub mod proof_data_handler; +pub mod pruning; pub mod query_eth_client; pub mod reorg_detector_checker; pub mod reorg_detector_runner; pub mod sigint; pub mod state_keeper; +pub mod sync_state_updater; pub mod tee_verifier_input_producer; +pub mod tree_data_fetcher; +pub mod validate_chain_ids; pub mod vm_runner; pub mod web3_api; diff --git a/core/node/node_framework/src/implementations/layers/postgres_metrics.rs b/core/node/node_framework/src/implementations/layers/postgres_metrics.rs new file mode 100644 index 000000000000..09d81844dd5a --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/postgres_metrics.rs @@ -0,0 +1,57 @@ +use std::time::Duration; + +use zksync_dal::{metrics::PostgresMetrics, ConnectionPool, Core}; + +use crate::{ + implementations::resources::pools::{PoolResource, ReplicaPool}, + service::{ServiceContext, StopReceiver}, + task::{TaskId, UnconstrainedTask}, + wiring_layer::{WiringError, WiringLayer}, +}; + +const SCRAPE_INTERVAL: Duration = Duration::from_secs(60); + +#[derive(Debug)] +pub struct PostgresMetricsLayer; + +#[async_trait::async_trait] +impl WiringLayer for PostgresMetricsLayer { + fn layer_name(&self) -> &'static str { + "postgres_metrics_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let replica_pool_resource = context.get_resource::>().await?; + let pool_for_metrics = replica_pool_resource.get_singleton().await?; + context.add_unconstrained_task(Box::new(PostgresMetricsScrapingTask { pool_for_metrics })); + + Ok(()) + } +} + +#[derive(Debug)] +struct PostgresMetricsScrapingTask { + pool_for_metrics: ConnectionPool, +} + +#[async_trait::async_trait] +impl UnconstrainedTask for PostgresMetricsScrapingTask { + fn id(&self) -> TaskId { + "postgres_metrics_scraping".into() + } + + async fn run_unconstrained( + self: Box, + mut stop_receiver: StopReceiver, + ) -> anyhow::Result<()> { + tokio::select! { + () = PostgresMetrics::run_scraping(self.pool_for_metrics, SCRAPE_INTERVAL) => { + tracing::warn!("Postgres metrics scraping unexpectedly stopped"); + } + _ = stop_receiver.0.changed() => { + tracing::info!("Stop signal received, Postgres metrics scraping is shutting down"); + } + } + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs b/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs index 6c7d4f915df4..4b7451348235 100644 --- a/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs +++ b/core/node/node_framework/src/implementations/layers/prometheus_exporter.rs @@ -4,7 +4,7 @@ use zksync_health_check::{HealthStatus, HealthUpdater, ReactiveHealthCheck}; use crate::{ implementations::resources::healthcheck::AppHealthCheckResource, service::{ServiceContext, StopReceiver}, - task::{Task, TaskId}, + task::{TaskId, UnconstrainedTask}, wiring_layer::{WiringError, WiringLayer}, }; @@ -43,18 +43,18 @@ impl WiringLayer for PrometheusExporterLayer { prometheus_health_updater, }); - node.add_task(task); + node.add_unconstrained_task(task); Ok(()) } } #[async_trait::async_trait] -impl Task for PrometheusExporterTask { +impl UnconstrainedTask for PrometheusExporterTask { fn id(&self) -> TaskId { "prometheus_exporter".into() } - async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn run_unconstrained(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { let prometheus_task = self.config.run(stop_receiver.0); self.prometheus_health_updater .update(HealthStatus::Ready.into()); diff --git a/core/node/node_framework/src/implementations/layers/pruning.rs b/core/node/node_framework/src/implementations/layers/pruning.rs new file mode 100644 index 000000000000..3ad52606083b --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/pruning.rs @@ -0,0 +1,75 @@ +use std::time::Duration; + +use zksync_node_db_pruner::{DbPruner, DbPrunerConfig}; + +use crate::{ + implementations::resources::{ + healthcheck::AppHealthCheckResource, + pools::{MasterPool, PoolResource}, + }, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct PruningLayer { + pruning_removal_delay: Duration, + pruning_chunk_size: u32, + minimum_l1_batch_age: Duration, +} + +impl PruningLayer { + pub fn new( + pruning_removal_delay: Duration, + pruning_chunk_size: u32, + minimum_l1_batch_age: Duration, + ) -> Self { + Self { + pruning_removal_delay, + pruning_chunk_size, + minimum_l1_batch_age, + } + } +} + +#[async_trait::async_trait] +impl WiringLayer for PruningLayer { + fn layer_name(&self) -> &'static str { + "pruning_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let pool_resource = context.get_resource::>().await?; + let main_pool = pool_resource.get().await?; + + let db_pruner = DbPruner::new( + DbPrunerConfig { + removal_delay: self.pruning_removal_delay, + pruned_batch_chunk_size: self.pruning_chunk_size, + minimum_l1_batch_age: self.minimum_l1_batch_age, + }, + main_pool, + ); + + let AppHealthCheckResource(app_health) = context.get_resource_or_default().await; + app_health + .insert_component(db_pruner.health_check()) + .map_err(WiringError::internal)?; + + context.add_task(Box::new(db_pruner)); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Task for DbPruner { + fn id(&self) -> TaskId { + "db_pruner".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs b/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs index 64454b63998b..eee63e6763b1 100644 --- a/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs +++ b/core/node/node_framework/src/implementations/layers/reorg_detector_checker.rs @@ -1,6 +1,7 @@ use std::time::Duration; use anyhow::Context; +use zksync_dal::{ConnectionPool, Core}; use zksync_reorg_detector::{self, ReorgDetector}; use crate::{ @@ -36,6 +37,7 @@ impl WiringLayer for ReorgDetectorCheckerLayer { // Create and insert precondition. context.add_precondition(Box::new(CheckerPrecondition { + pool: pool.clone(), reorg_detector: ReorgDetector::new(main_node_client, pool), })); @@ -44,6 +46,7 @@ impl WiringLayer for ReorgDetectorCheckerLayer { } pub struct CheckerPrecondition { + pool: ConnectionPool, reorg_detector: ReorgDetector, } @@ -53,7 +56,21 @@ impl Precondition for CheckerPrecondition { "reorg_detector_checker".into() } - async fn check(mut self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + async fn check(mut self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + // Given that this is a precondition -- i.e. something that starts before some invariants are met, + // we need to first ensure that there is at least one batch in the database (there may be none if + // either genesis or snapshot recovery has not been performed yet). + let earliest_batch = zksync_dal::helpers::wait_for_l1_batch( + &self.pool, + REORG_DETECTED_SLEEP_INTERVAL, + &mut stop_receiver.0, + ) + .await?; + if earliest_batch.is_none() { + // Stop signal received. + return Ok(()); + } + loop { match self.reorg_detector.run_once(stop_receiver.0.clone()).await { Ok(()) => return Ok(()), diff --git a/core/node/node_framework/src/implementations/layers/state_keeper/external_io.rs b/core/node/node_framework/src/implementations/layers/state_keeper/external_io.rs new file mode 100644 index 000000000000..1ec80fef4272 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/state_keeper/external_io.rs @@ -0,0 +1,68 @@ +use std::sync::Arc; + +use anyhow::Context as _; +use zksync_node_sync::{ActionQueue, ExternalIO, SyncState}; +use zksync_state_keeper::seal_criteria::NoopSealer; +use zksync_types::L2ChainId; + +use crate::{ + implementations::resources::{ + action_queue::ActionQueueSenderResource, + main_node_client::MainNodeClientResource, + pools::{MasterPool, PoolResource}, + state_keeper::{ConditionalSealerResource, StateKeeperIOResource}, + sync_state::SyncStateResource, + }, + resource::Unique, + service::ServiceContext, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct ExternalIOLayer { + chain_id: L2ChainId, +} + +impl ExternalIOLayer { + pub fn new(chain_id: L2ChainId) -> Self { + Self { chain_id } + } +} + +#[async_trait::async_trait] +impl WiringLayer for ExternalIOLayer { + fn layer_name(&self) -> &'static str { + "external_io_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + // Fetch required resources. + let master_pool = context.get_resource::>().await?; + let MainNodeClientResource(main_node_client) = context.get_resource().await?; + + // Create `SyncState` resource. + let sync_state = SyncState::default(); + context.insert_resource(SyncStateResource(sync_state))?; + + // Create `ActionQueueSender` resource. + let (action_queue_sender, action_queue) = ActionQueue::new(); + context.insert_resource(ActionQueueSenderResource(Unique::new(action_queue_sender)))?; + + // Create external IO resource. + let io_pool = master_pool.get().await.context("Get master pool")?; + let io = ExternalIO::new( + io_pool, + action_queue, + Box::new(main_node_client.for_component("external_io")), + self.chain_id, + ) + .await + .context("Failed initializing I/O for external node state keeper")?; + context.insert_resource(StateKeeperIOResource(Unique::new(Box::new(io))))?; + + // Create sealer. + context.insert_resource(ConditionalSealerResource(Arc::new(NoopSealer)))?; + + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/state_keeper/main_batch_executor.rs b/core/node/node_framework/src/implementations/layers/state_keeper/main_batch_executor.rs index 2fb35fb201ab..82e6e52274aa 100644 --- a/core/node/node_framework/src/implementations/layers/state_keeper/main_batch_executor.rs +++ b/core/node/node_framework/src/implementations/layers/state_keeper/main_batch_executor.rs @@ -1,4 +1,3 @@ -use zksync_config::configs::chain::StateKeeperConfig; use zksync_state_keeper::MainBatchExecutor; use crate::{ @@ -10,13 +9,15 @@ use crate::{ #[derive(Debug)] pub struct MainBatchExecutorLayer { - state_keeper_config: StateKeeperConfig, + save_call_traces: bool, + optional_bytecode_compression: bool, } impl MainBatchExecutorLayer { - pub fn new(state_keeper_config: StateKeeperConfig) -> Self { + pub fn new(save_call_traces: bool, optional_bytecode_compression: bool) -> Self { Self { - state_keeper_config, + save_call_traces, + optional_bytecode_compression, } } } @@ -28,7 +29,8 @@ impl WiringLayer for MainBatchExecutorLayer { } async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { - let builder = MainBatchExecutor::new(self.state_keeper_config.save_call_traces, false); + let builder = + MainBatchExecutor::new(self.save_call_traces, self.optional_bytecode_compression); context.insert_resource(BatchExecutorResource(Unique::new(Box::new(builder))))?; Ok(()) diff --git a/core/node/node_framework/src/implementations/layers/state_keeper/mempool_io.rs b/core/node/node_framework/src/implementations/layers/state_keeper/mempool_io.rs index 65e86bef5204..1a913fd990bf 100644 --- a/core/node/node_framework/src/implementations/layers/state_keeper/mempool_io.rs +++ b/core/node/node_framework/src/implementations/layers/state_keeper/mempool_io.rs @@ -1,24 +1,18 @@ use std::sync::Arc; use anyhow::Context as _; -use zksync_config::{ - configs::{ - chain::{MempoolConfig, StateKeeperConfig}, - wallets, - }, - ContractsConfig, -}; -use zksync_state_keeper::{ - io::seal_logic::l2_block_seal_subtasks::L2BlockSealProcess, MempoolFetcher, MempoolGuard, - MempoolIO, OutputHandler, SequencerSealer, StateKeeperPersistence, TreeWritesPersistence, +use zksync_config::configs::{ + chain::{MempoolConfig, StateKeeperConfig}, + wallets, }; +use zksync_state_keeper::{MempoolFetcher, MempoolGuard, MempoolIO, SequencerSealer}; use zksync_types::L2ChainId; use crate::{ implementations::resources::{ fee_input::FeeInputResource, pools::{MasterPool, PoolResource}, - state_keeper::{ConditionalSealerResource, OutputHandlerResource, StateKeeperIOResource}, + state_keeper::{ConditionalSealerResource, StateKeeperIOResource}, }, resource::Unique, service::{ServiceContext, StopReceiver}, @@ -29,7 +23,6 @@ use crate::{ #[derive(Debug)] pub struct MempoolIOLayer { zksync_network_id: L2ChainId, - contracts_config: ContractsConfig, state_keeper_config: StateKeeperConfig, mempool_config: MempoolConfig, wallets: wallets::StateKeeper, @@ -38,14 +31,12 @@ pub struct MempoolIOLayer { impl MempoolIOLayer { pub fn new( zksync_network_id: L2ChainId, - contracts_config: ContractsConfig, state_keeper_config: StateKeeperConfig, mempool_config: MempoolConfig, wallets: wallets::StateKeeper, ) -> Self { Self { zksync_network_id, - contracts_config, state_keeper_config, mempool_config, wallets, @@ -81,23 +72,6 @@ impl WiringLayer for MempoolIOLayer { let batch_fee_input_provider = context.get_resource::().await?.0; let master_pool = context.get_resource::>().await?; - // Create L2 block sealer task and output handler. - // L2 Block sealing process is parallelized, so we have to provide enough pooled connections. - let persistence_pool = master_pool - .get_custom(L2BlockSealProcess::subtasks_len()) - .await - .context("Get master pool")?; - let (persistence, l2_block_sealer) = StateKeeperPersistence::new( - persistence_pool.clone(), - self.contracts_config.l2_shared_bridge_addr.unwrap(), - self.state_keeper_config.l2_block_seal_queue_capacity, - ); - let tree_writes_persistence = TreeWritesPersistence::new(persistence_pool); - let output_handler = OutputHandler::new(Box::new(persistence)) - .with_handler(Box::new(tree_writes_persistence)); - context.insert_resource(OutputHandlerResource(Unique::new(output_handler)))?; - context.add_task(Box::new(L2BlockSealerTask(l2_block_sealer))); - // Create mempool fetcher task. let mempool_guard = self.build_mempool_guard(&master_pool).await?; let mempool_fetcher_pool = master_pool @@ -137,21 +111,6 @@ impl WiringLayer for MempoolIOLayer { } } -#[derive(Debug)] -struct L2BlockSealerTask(zksync_state_keeper::L2BlockSealerTask); - -#[async_trait::async_trait] -impl Task for L2BlockSealerTask { - fn id(&self) -> TaskId { - "state_keeper/l2_block_sealer".into() - } - - async fn run(self: Box, _stop_receiver: StopReceiver) -> anyhow::Result<()> { - // Miniblock sealer will exit itself once sender is dropped. - self.0.run().await - } -} - #[derive(Debug)] struct MempoolFetcherTask(MempoolFetcher); diff --git a/core/node/node_framework/src/implementations/layers/state_keeper/mod.rs b/core/node/node_framework/src/implementations/layers/state_keeper/mod.rs index edbe1d6e12f7..97364f6388cd 100644 --- a/core/node/node_framework/src/implementations/layers/state_keeper/mod.rs +++ b/core/node/node_framework/src/implementations/layers/state_keeper/mod.rs @@ -1,16 +1,20 @@ use std::sync::Arc; use anyhow::Context; -use zksync_config::DBConfig; -use zksync_state::{AsyncCatchupTask, ReadStorageFactory, RocksdbStorageOptions}; +use zksync_state::{AsyncCatchupTask, ReadStorageFactory}; use zksync_state_keeper::{ seal_criteria::ConditionalSealer, AsyncRocksdbCache, BatchExecutor, OutputHandler, StateKeeperIO, ZkSyncStateKeeper, }; use zksync_storage::RocksDB; +pub mod external_io; pub mod main_batch_executor; pub mod mempool_io; +pub mod output_handler; + +// Public re-export to not require the user to directly depend on `zksync_state`. +pub use zksync_state::RocksdbStorageOptions; use crate::{ implementations::resources::{ @@ -32,12 +36,16 @@ use crate::{ /// #[derive(Debug)] pub struct StateKeeperLayer { - db_config: DBConfig, + state_keeper_db_path: String, + rocksdb_options: RocksdbStorageOptions, } impl StateKeeperLayer { - pub fn new(db_config: DBConfig) -> Self { - Self { db_config } + pub fn new(state_keeper_db_path: String, rocksdb_options: RocksdbStorageOptions) -> Self { + Self { + state_keeper_db_path, + rocksdb_options, + } } } @@ -69,17 +77,10 @@ impl WiringLayer for StateKeeperLayer { let sealer = context.get_resource::().await?.0; let master_pool = context.get_resource::>().await?; - let cache_options = RocksdbStorageOptions { - block_cache_capacity: self - .db_config - .experimental - .state_keeper_db_block_cache_capacity(), - max_open_files: self.db_config.experimental.state_keeper_db_max_open_files, - }; let (storage_factory, task) = AsyncRocksdbCache::new( master_pool.get_custom(2).await?, - self.db_config.state_keeper_db_path, - cache_options, + self.state_keeper_db_path, + self.rocksdb_options, ); context.add_task(Box::new(RocksdbCatchupTask(task))); diff --git a/core/node/node_framework/src/implementations/layers/state_keeper/output_handler.rs b/core/node/node_framework/src/implementations/layers/state_keeper/output_handler.rs new file mode 100644 index 000000000000..d0e94f637e08 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/state_keeper/output_handler.rs @@ -0,0 +1,121 @@ +use anyhow::Context as _; +use zksync_state_keeper::{ + io::seal_logic::l2_block_seal_subtasks::L2BlockSealProcess, OutputHandler, + StateKeeperPersistence, TreeWritesPersistence, +}; +use zksync_types::Address; + +use crate::{ + implementations::resources::{ + pools::{MasterPool, PoolResource}, + state_keeper::OutputHandlerResource, + sync_state::SyncStateResource, + }, + resource::Unique, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct OutputHandlerLayer { + l2_shared_bridge_addr: Address, + l2_block_seal_queue_capacity: usize, + /// Whether transactions should be pre-inserted to DB. + /// Should be set to `true` for EN's IO as EN doesn't store transactions in DB + /// before they are included into L2 blocks. + pre_insert_txs: bool, + /// Whether protective reads persistence is enabled. + /// Must be `true` for any node that maintains a full Merkle Tree (e.g. any instance of main node). + /// May be set to `false` for nodes that do not participate in the sequencing process (e.g. external nodes). + protective_reads_persistence_enabled: bool, +} + +impl OutputHandlerLayer { + pub fn new(l2_shared_bridge_addr: Address, l2_block_seal_queue_capacity: usize) -> Self { + Self { + l2_shared_bridge_addr, + l2_block_seal_queue_capacity, + pre_insert_txs: false, + protective_reads_persistence_enabled: true, + } + } + + pub fn with_pre_insert_txs(mut self, pre_insert_txs: bool) -> Self { + self.pre_insert_txs = pre_insert_txs; + self + } + + pub fn with_protective_reads_persistence_enabled( + mut self, + protective_reads_persistence_enabled: bool, + ) -> Self { + self.protective_reads_persistence_enabled = protective_reads_persistence_enabled; + self + } +} + +#[async_trait::async_trait] +impl WiringLayer for OutputHandlerLayer { + fn layer_name(&self) -> &'static str { + "state_keeper_output_handler_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + // Fetch required resources. + let master_pool = context.get_resource::>().await?; + // Use `SyncState` if provided. + let sync_state = match context.get_resource::().await { + Ok(sync_state) => Some(sync_state.0), + Err(WiringError::ResourceLacking { .. }) => None, + Err(err) => return Err(err), + }; + + // Create L2 block sealer task and output handler. + // L2 Block sealing process is parallelized, so we have to provide enough pooled connections. + let persistence_pool = master_pool + .get_custom(L2BlockSealProcess::subtasks_len()) + .await + .context("Get master pool")?; + let (mut persistence, l2_block_sealer) = StateKeeperPersistence::new( + persistence_pool.clone(), + self.l2_shared_bridge_addr, + self.l2_block_seal_queue_capacity, + ); + if self.pre_insert_txs { + persistence = persistence.with_tx_insertion(); + } + if !self.protective_reads_persistence_enabled { + // **Important:** Disabling protective reads persistence is only sound if the node will never + // run a full Merkle tree. + tracing::warn!("Disabling persisting protective reads; this should be safe, but is considered an experimental option at the moment"); + persistence = persistence.without_protective_reads(); + } + + let tree_writes_persistence = TreeWritesPersistence::new(persistence_pool); + let mut output_handler = OutputHandler::new(Box::new(persistence)) + .with_handler(Box::new(tree_writes_persistence)); + if let Some(sync_state) = sync_state { + output_handler = output_handler.with_handler(Box::new(sync_state)); + } + context.insert_resource(OutputHandlerResource(Unique::new(output_handler)))?; + context.add_task(Box::new(L2BlockSealerTask(l2_block_sealer))); + + Ok(()) + } +} + +#[derive(Debug)] +struct L2BlockSealerTask(zksync_state_keeper::L2BlockSealerTask); + +#[async_trait::async_trait] +impl Task for L2BlockSealerTask { + fn id(&self) -> TaskId { + "state_keeper/l2_block_sealer".into() + } + + async fn run(self: Box, _stop_receiver: StopReceiver) -> anyhow::Result<()> { + // Miniblock sealer will exit itself once sender is dropped. + self.0.run().await + } +} diff --git a/core/node/node_framework/src/implementations/layers/sync_state_updater.rs b/core/node/node_framework/src/implementations/layers/sync_state_updater.rs new file mode 100644 index 000000000000..fcbe51f581e1 --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/sync_state_updater.rs @@ -0,0 +1,75 @@ +use zksync_dal::{ConnectionPool, Core}; +use zksync_node_sync::SyncState; +use zksync_web3_decl::client::{DynClient, L2}; + +use crate::{ + implementations::resources::{ + main_node_client::MainNodeClientResource, + pools::{MasterPool, PoolResource}, + sync_state::SyncStateResource, + }, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +/// Runs the dynamic sync state updater for `SyncState` if no `SyncState` was provided before. +/// This layer may be used as a fallback for EN API if API server runs without the core component. +#[derive(Debug)] +pub struct SyncStateUpdaterLayer; + +#[async_trait::async_trait] +impl WiringLayer for SyncStateUpdaterLayer { + fn layer_name(&self) -> &'static str { + "sync_state_updater_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + if context.get_resource::().await.is_ok() { + // `SyncState` was provided by some other layer -- we assume that the layer that added this resource + // will be responsible for its maintenance. + tracing::info!( + "SyncState was provided by another layer, skipping SyncStateUpdaterLayer" + ); + return Ok(()); + } + + let pool = context.get_resource::>().await?; + let MainNodeClientResource(main_node_client) = context.get_resource().await?; + + let sync_state = SyncState::default(); + + // Insert resource. + context.insert_resource(SyncStateResource(sync_state.clone()))?; + + // Insert task + context.add_task(Box::new(SyncStateUpdater { + sync_state, + connection_pool: pool.get().await?, + main_node_client, + })); + + Ok(()) + } +} + +#[derive(Debug)] +struct SyncStateUpdater { + sync_state: SyncState, + connection_pool: ConnectionPool, + main_node_client: Box>, +} + +#[async_trait::async_trait] +impl Task for SyncStateUpdater { + fn id(&self) -> TaskId { + "sync_state_updater".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + self.sync_state + .run_updater(self.connection_pool, self.main_node_client, stop_receiver.0) + .await?; + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/tree_data_fetcher.rs b/core/node/node_framework/src/implementations/layers/tree_data_fetcher.rs new file mode 100644 index 000000000000..c45071ce418b --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/tree_data_fetcher.rs @@ -0,0 +1,67 @@ +use zksync_node_sync::tree_data_fetcher::TreeDataFetcher; +use zksync_types::Address; + +use crate::{ + implementations::resources::{ + eth_interface::EthInterfaceResource, + healthcheck::AppHealthCheckResource, + main_node_client::MainNodeClientResource, + pools::{MasterPool, PoolResource}, + }, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct TreeDataFetcherLayer { + diamond_proxy_addr: Address, +} + +impl TreeDataFetcherLayer { + pub fn new(diamond_proxy_addr: Address) -> Self { + Self { diamond_proxy_addr } + } +} + +#[async_trait::async_trait] +impl WiringLayer for TreeDataFetcherLayer { + fn layer_name(&self) -> &'static str { + "tree_data_fetcher_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let pool = context.get_resource::>().await?; + let MainNodeClientResource(client) = context.get_resource().await?; + let EthInterfaceResource(eth_client) = context.get_resource().await?; + + tracing::warn!( + "Running tree data fetcher (allows a node to operate w/o a Merkle tree or w/o waiting the tree to catch up). \ + This is an experimental feature; do not use unless you know what you're doing" + ); + let fetcher = TreeDataFetcher::new(client, pool.get().await?) + .with_l1_data(eth_client, self.diamond_proxy_addr)?; + + // Insert healthcheck + let AppHealthCheckResource(app_health) = context.get_resource_or_default().await; + app_health + .insert_component(fetcher.health_check()) + .map_err(WiringError::internal)?; + + // Insert task + context.add_task(Box::new(fetcher)); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Task for TreeDataFetcher { + fn id(&self) -> TaskId { + "tree_data_fetcher".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs b/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs new file mode 100644 index 000000000000..0f04a35d484a --- /dev/null +++ b/core/node/node_framework/src/implementations/layers/validate_chain_ids.rs @@ -0,0 +1,61 @@ +use zksync_node_sync::validate_chain_ids_task::ValidateChainIdsTask; +use zksync_types::{L1ChainId, L2ChainId}; + +use crate::{ + implementations::resources::{ + eth_interface::EthInterfaceResource, main_node_client::MainNodeClientResource, + }, + precondition::Precondition, + service::{ServiceContext, StopReceiver}, + task::TaskId, + wiring_layer::{WiringError, WiringLayer}, +}; + +#[derive(Debug)] +pub struct ValidateChainIdsLayer { + l1_chain_id: L1ChainId, + l2_chain_id: L2ChainId, +} + +impl ValidateChainIdsLayer { + pub fn new(l1_chain_id: L1ChainId, l2_chain_id: L2ChainId) -> Self { + Self { + l1_chain_id, + l2_chain_id, + } + } +} + +#[async_trait::async_trait] +impl WiringLayer for ValidateChainIdsLayer { + fn layer_name(&self) -> &'static str { + "validate_chain_ids_layer" + } + + async fn wire(self: Box, mut context: ServiceContext<'_>) -> Result<(), WiringError> { + let EthInterfaceResource(query_client) = context.get_resource().await?; + let MainNodeClientResource(main_node_client) = context.get_resource().await?; + + let task = ValidateChainIdsTask::new( + self.l1_chain_id, + self.l2_chain_id, + query_client, + main_node_client, + ); + + context.add_precondition(Box::new(task)); + + Ok(()) + } +} + +#[async_trait::async_trait] +impl Precondition for ValidateChainIdsTask { + fn id(&self) -> TaskId { + "validate_chain_ids".into() + } + + async fn check(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run_once(stop_receiver.0).await + } +} diff --git a/core/node/node_framework/src/implementations/layers/web3_api/server.rs b/core/node/node_framework/src/implementations/layers/web3_api/server.rs index c81b475c3ec4..da0d9d3cc33a 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/server.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/server.rs @@ -27,8 +27,11 @@ pub struct Web3ServerOptionalConfig { pub batch_request_size_limit: Option, pub response_body_size_limit: Option, pub websocket_requests_per_minute_limit: Option, - // used by circuit breaker. + pub with_extended_tracing: bool, + // Used by circuit breaker. pub replication_lag_limit: Option, + // Used by the external node. + pub pruning_info_refresh_interval: Option, } impl Web3ServerOptionalConfig { @@ -132,7 +135,8 @@ impl WiringLayer for Web3ServerLayer { ApiBuilder::jsonrpsee_backend(self.internal_api_config, replica_pool.clone()) .with_updaters_pool(updaters_pool) .with_tx_sender(tx_sender) - .with_mempool_cache(mempool_cache); + .with_mempool_cache(mempool_cache) + .with_extended_tracing(self.optional_config.with_extended_tracing); if let Some(client) = tree_api_client { api_builder = api_builder.with_tree_api(client); } @@ -147,6 +151,12 @@ impl WiringLayer for Web3ServerLayer { if let Some(sync_state) = sync_state { api_builder = api_builder.with_sync_state(sync_state); } + if let Some(pruning_info_refresh_interval) = + self.optional_config.pruning_info_refresh_interval + { + api_builder = + api_builder.with_pruning_info_refresh_interval(pruning_info_refresh_interval); + } let replication_lag_limit = self.optional_config.replication_lag_limit; api_builder = self.optional_config.apply(api_builder); let server = api_builder.build()?; diff --git a/core/node/node_framework/src/implementations/layers/web3_api/tx_sender.rs b/core/node/node_framework/src/implementations/layers/web3_api/tx_sender.rs index 8a717258cb46..010778315e58 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/tx_sender.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/tx_sender.rs @@ -1,14 +1,22 @@ -use std::{fmt, sync::Arc}; +use std::{fmt, sync::Arc, time::Duration}; +use tokio::sync::RwLock; use zksync_node_api_server::{ execution_sandbox::{VmConcurrencyBarrier, VmConcurrencyLimiter}, tx_sender::{ApiContracts, TxSenderBuilder, TxSenderConfig}, }; use zksync_state::PostgresStorageCaches; +use zksync_types::Address; +use zksync_web3_decl::{ + client::{DynClient, L2}, + jsonrpsee, + namespaces::EnNamespaceClient as _, +}; use crate::{ implementations::resources::{ fee_input::FeeInputResource, + main_node_client::MainNodeClientResource, pools::{PoolResource, ReplicaPool}, state_keeper::ConditionalSealerResource, web3_api::{TxSenderResource, TxSinkResource}, @@ -31,6 +39,7 @@ pub struct TxSenderLayer { postgres_storage_caches_config: PostgresStorageCachesConfig, max_vm_concurrency: usize, api_contracts: ApiContracts, + whitelisted_tokens_for_aa_cache: bool, } impl TxSenderLayer { @@ -45,8 +54,18 @@ impl TxSenderLayer { postgres_storage_caches_config, max_vm_concurrency, api_contracts, + whitelisted_tokens_for_aa_cache: false, } } + + /// Enables the task for fetching the whitelisted tokens for the AA cache from the main node. + /// Disabled by default. + /// + /// Requires `MainNodeClientResource` to be present. + pub fn with_whitelisted_tokens_for_aa_cache(mut self, value: bool) -> Self { + self.whitelisted_tokens_for_aa_cache = value; + self + } } #[async_trait::async_trait] @@ -96,6 +115,18 @@ impl WiringLayer for TxSenderLayer { if let Some(sealer) = sealer { tx_sender = tx_sender.with_sealer(sealer); } + + // Add the task for updating the whitelisted tokens for the AA cache. + if self.whitelisted_tokens_for_aa_cache { + let MainNodeClientResource(main_node_client) = context.get_resource().await?; + let whitelisted_tokens = Arc::new(RwLock::new(Default::default())); + context.add_task(Box::new(WhitelistedTokensForAaUpdateTask { + whitelisted_tokens: whitelisted_tokens.clone(), + main_node_client, + })); + tx_sender = tx_sender.with_whitelisted_tokens_for_aa(whitelisted_tokens); + } + let tx_sender = tx_sender.build( fee_input, Arc::new(vm_concurrency_limiter), @@ -153,3 +184,40 @@ impl Task for VmConcurrencyBarrierTask { Ok(()) } } + +#[derive(Debug)] +struct WhitelistedTokensForAaUpdateTask { + whitelisted_tokens: Arc>>, + main_node_client: Box>, +} + +#[async_trait::async_trait] +impl Task for WhitelistedTokensForAaUpdateTask { + fn id(&self) -> TaskId { + "whitelisted_tokens_for_aa_update_task".into() + } + + async fn run(mut self: Box, mut stop_receiver: StopReceiver) -> anyhow::Result<()> { + while !*stop_receiver.0.borrow_and_update() { + match self.main_node_client.whitelisted_tokens_for_aa().await { + Ok(tokens) => { + *self.whitelisted_tokens.write().await = tokens; + } + Err(jsonrpsee::core::client::Error::Call(error)) + if error.code() == jsonrpsee::types::error::METHOD_NOT_FOUND_CODE => + { + // Method is not supported by the main node, do nothing. + } + Err(err) => { + tracing::error!("Failed to query `whitelisted_tokens_for_aa`, error: {err:?}"); + } + } + + // Error here corresponds to a timeout w/o `stop_receiver` changed; we're OK with this. + tokio::time::timeout(Duration::from_secs(30), stop_receiver.0.changed()) + .await + .ok(); + } + Ok(()) + } +} diff --git a/core/node/node_framework/src/implementations/layers/web3_api/tx_sink.rs b/core/node/node_framework/src/implementations/layers/web3_api/tx_sink.rs index df4812b3c098..98ed50ba9e45 100644 --- a/core/node/node_framework/src/implementations/layers/web3_api/tx_sink.rs +++ b/core/node/node_framework/src/implementations/layers/web3_api/tx_sink.rs @@ -1,6 +1,9 @@ use std::sync::Arc; -use zksync_node_api_server::tx_sender::{master_pool_sink::MasterPoolSink, proxy::TxProxy}; +use zksync_node_api_server::tx_sender::{ + master_pool_sink::MasterPoolSink, + proxy::{AccountNonceSweeperTask, TxProxy}, +}; use crate::{ implementations::resources::{ @@ -8,7 +11,8 @@ use crate::{ pools::{MasterPool, PoolResource}, web3_api::TxSinkResource, }, - service::ServiceContext, + service::{ServiceContext, StopReceiver}, + task::{Task, TaskId}, wiring_layer::{WiringError, WiringLayer}, }; @@ -37,10 +41,31 @@ impl WiringLayer for TxSinkLayer { } TxSinkLayer::ProxySink => { let MainNodeClientResource(client) = context.get_resource().await?; - TxSinkResource(Arc::new(TxProxy::new(client))) + let proxy = TxProxy::new(client); + + let pool = context + .get_resource::>() + .await? + .get_singleton() + .await?; + let task = proxy.account_nonce_sweeper_task(pool); + context.add_task(Box::new(task)); + + TxSinkResource(Arc::new(proxy)) } }; context.insert_resource(tx_sink)?; Ok(()) } } + +#[async_trait::async_trait] +impl Task for AccountNonceSweeperTask { + fn id(&self) -> TaskId { + "account_nonce_sweeper_task".into() + } + + async fn run(self: Box, stop_receiver: StopReceiver) -> anyhow::Result<()> { + (*self).run(stop_receiver.0).await + } +} diff --git a/core/node/node_sync/src/validate_chain_ids_task.rs b/core/node/node_sync/src/validate_chain_ids_task.rs index 5a75cb384aec..1414b5ab6014 100644 --- a/core/node/node_sync/src/validate_chain_ids_task.rs +++ b/core/node/node_sync/src/validate_chain_ids_task.rs @@ -138,6 +138,23 @@ impl ValidateChainIdsTask { } } + /// Runs the task once, exiting either when all the checks are performed or when the stop signal is received. + pub async fn run_once(self, mut stop_receiver: watch::Receiver) -> anyhow::Result<()> { + let eth_client_check = Self::check_eth_client(self.eth_client, self.l1_chain_id); + let main_node_l1_check = + Self::check_l1_chain_using_main_node(self.main_node_client.clone(), self.l1_chain_id); + let main_node_l2_check = + Self::check_l2_chain_using_main_node(self.main_node_client, self.l2_chain_id); + let joined_futures = + futures::future::try_join3(eth_client_check, main_node_l1_check, main_node_l2_check) + .fuse(); + tokio::select! { + res = joined_futures => res.map(drop), + _ = stop_receiver.changed() => Ok(()), + } + } + + /// Runs the task until the stop signal is received. pub async fn run(self, mut stop_receiver: watch::Receiver) -> anyhow::Result<()> { // Since check futures are fused, they are safe to poll after getting resolved; they will never resolve again, // so we'll just wait for another check or a stop signal. diff --git a/core/node/state_keeper/src/batch_executor/main_executor.rs b/core/node/state_keeper/src/batch_executor/main_executor.rs index a16b9920dd6e..f3f947d0d1e6 100644 --- a/core/node/state_keeper/src/batch_executor/main_executor.rs +++ b/core/node/state_keeper/src/batch_executor/main_executor.rs @@ -32,6 +32,12 @@ use crate::{ #[derive(Debug, Clone)] pub struct MainBatchExecutor { save_call_traces: bool, + /// Whether batch executor would allow transactions with bytecode that cannot be compressed. + /// For new blocks, bytecode compression is mandatory -- if bytecode compression is not supported, + /// the transaction will be rejected. + /// Note that this flag, if set to `true`, is strictly more permissive than if set to `false`. It means + /// that in cases where the node is expected to process any transactions processed by the sequencer + /// regardless of its configuration, this flag should be set to `true`. optional_bytecode_compression: bool, } @@ -218,6 +224,8 @@ impl CommandReceiver { result } + /// Attempts to execute transaction with or without bytecode compression. + /// If compression fails, the transaction will be re-executed without compression. fn execute_tx_in_vm_with_optional_compression( &self, tx: &Transaction, @@ -283,10 +291,8 @@ impl CommandReceiver { (result.1, compressed_bytecodes, trace) } - // Err when transaction is rejected. - // `Ok(TxExecutionStatus::Success)` when the transaction succeeded - // `Ok(TxExecutionStatus::Failure)` when the transaction failed. - // Note that failed transactions are considered properly processed and are included in blocks + /// Attempts to execute transaction with mandatory bytecode compression. + /// If bytecode compression fails, the transaction will be rejected. fn execute_tx_in_vm( &self, tx: &Transaction, diff --git a/etc/env/configs/ext-node.toml b/etc/env/configs/ext-node.toml index eb07aa387542..145b1455ab93 100644 --- a/etc/env/configs/ext-node.toml +++ b/etc/env/configs/ext-node.toml @@ -55,6 +55,8 @@ url = "http://127.0.0.1:3050" # Here we use TOML multiline strings: newlines will be trimmed. log = """\ warn,\ +zksync_node_framework=info,\ +zksync_node_consensus=info,\ zksync_consensus_bft=info,\ zksync_consensus_network=info,\ zksync_consensus_storage=info,\ From 3ee34be7e48fb4b7c5030a6422a0a9f8a8ebc35b Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 19 Jun 2024 13:42:47 +0300 Subject: [PATCH 20/43] perf(db): Try yet another storage log pruning approach (#2268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Structures storage log pruning differently by first loading primary keys for the latest logs in the pruned block range, and then range-removing older logs based on these PKs. Both of these queries are designed to use particular indexes, making them have predictable performance. ## Why ❔ The current DB queries for storage log pruning sometimes use unpredictable indexes and have suboptimal performance. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- ...60cd2f3d5223add676591cb0577e0a77403cb.json | 16 +++ ...6ba34fd131682ee5414a9d0ae2cab349b2395.json | 15 --- ...0e8a100140875f95cd8cf5de3c6202d59a19c.json | 15 --- ...94d8d631d56c5753f4e944f1cdf3e05b04a8c.json | 35 ++++++ core/lib/dal/src/pruning_dal/mod.rs | 119 +++++++++--------- core/lib/dal/src/pruning_dal/tests.rs | 6 +- core/node/db_pruner/src/metrics.rs | 19 ++- 7 files changed, 123 insertions(+), 102 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json delete mode 100644 core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json delete mode 100644 core/lib/dal/.sqlx/query-4cff62fad4a7044a824a60656050e8a100140875f95cd8cf5de3c6202d59a19c.json create mode 100644 core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json diff --git a/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json b/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json new file mode 100644 index 000000000000..7ecce5be1f35 --- /dev/null +++ b/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "\n DELETE FROM storage_logs USING UNNEST($1::bytea[], $2::BIGINT[], $3::INT[]) AS new_logs (hashed_key, miniblock_number, operation_number)\n WHERE\n storage_logs.hashed_key = new_logs.hashed_key\n AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "ByteaArray", + "Int8Array", + "Int4Array" + ] + }, + "nullable": [] + }, + "hash": "327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb" +} diff --git a/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json b/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json deleted file mode 100644 index ef84a26a6e84..000000000000 --- a/core/lib/dal/.sqlx/query-362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM storage_logs\n WHERE\n storage_logs.miniblock_number < $1\n AND hashed_key IN (\n SELECT\n hashed_key\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "362e20c4c2527f1585132ca85316ba34fd131682ee5414a9d0ae2cab349b2395" -} diff --git a/core/lib/dal/.sqlx/query-4cff62fad4a7044a824a60656050e8a100140875f95cd8cf5de3c6202d59a19c.json b/core/lib/dal/.sqlx/query-4cff62fad4a7044a824a60656050e8a100140875f95cd8cf5de3c6202d59a19c.json deleted file mode 100644 index 2c4d795f2f45..000000000000 --- a/core/lib/dal/.sqlx/query-4cff62fad4a7044a824a60656050e8a100140875f95cd8cf5de3c6202d59a19c.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM storage_logs USING (\n SELECT\n hashed_key,\n MAX(ARRAY[miniblock_number, operation_number]::INT[]) AS op\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n GROUP BY\n hashed_key\n ) AS last_storage_logs\n WHERE\n storage_logs.miniblock_number BETWEEN $1 AND $2\n AND last_storage_logs.hashed_key = storage_logs.hashed_key\n AND (\n storage_logs.miniblock_number != last_storage_logs.op[1]\n OR storage_logs.operation_number != last_storage_logs.op[2]\n )\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "4cff62fad4a7044a824a60656050e8a100140875f95cd8cf5de3c6202d59a19c" -} diff --git a/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json b/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json new file mode 100644 index 000000000000..ffb51e0dd865 --- /dev/null +++ b/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json @@ -0,0 +1,35 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT\n ON (hashed_key) hashed_key,\n miniblock_number,\n operation_number\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n ORDER BY\n hashed_key,\n miniblock_number DESC,\n operation_number DESC\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "hashed_key", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "miniblock_number", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "operation_number", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [ + false, + false, + false + ] + }, + "hash": "8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c" +} diff --git a/core/lib/dal/src/pruning_dal/mod.rs b/core/lib/dal/src/pruning_dal/mod.rs index 9a5356202aee..16f85f2e0fad 100644 --- a/core/lib/dal/src/pruning_dal/mod.rs +++ b/core/lib/dal/src/pruning_dal/mod.rs @@ -1,5 +1,6 @@ use std::ops; +use itertools::Itertools; use zksync_db_connection::{connection::Connection, error::DalResult, instrument::InstrumentExt}; use zksync_types::{L1BatchNumber, L2BlockNumber}; @@ -27,8 +28,8 @@ pub struct PruningInfo { pub struct HardPruningStats { pub deleted_l1_batches: u64, pub deleted_l2_blocks: u64, - pub deleted_storage_logs_from_past_batches: u64, - pub deleted_storage_logs_from_pruned_batches: u64, + pub overwriting_logs: u64, + pub deleted_storage_logs: u64, pub deleted_events: u64, pub deleted_call_traces: u64, pub deleted_l2_to_l1_logs: u64, @@ -41,6 +42,14 @@ enum PruneType { Hard, } +/// Raw database presentation of a primary key in the `miniblocks` table. +#[derive(Debug)] +struct StorageLogPrimaryKey { + hashed_key: Vec, + miniblock_number: i64, + operation_number: i32, +} + impl PruningDal<'_, '_> { pub async fn get_pruning_info(&mut self) -> DalResult { let pruning_info = sqlx::query!( @@ -174,17 +183,18 @@ impl PruningDal<'_, '_> { self.clear_transaction_fields(first_l2_block_to_prune..=last_l2_block_to_prune) .await?; - // The deleting of logs is split into two queries to make it faster, - // only the first query has to go through all previous logs - // and the query optimizer should be happy with it - let deleted_storage_logs_from_past_batches = self - .prune_storage_logs_from_past_l2_blocks( - first_l2_block_to_prune..=last_l2_block_to_prune, - ) - .await?; - let deleted_storage_logs_from_pruned_batches = self - .prune_storage_logs_in_range(first_l2_block_to_prune..=last_l2_block_to_prune) + // Storage log pruning is designed to use deterministic indexes and thus have predictable performance. + // + // - `get_pks_for_latest_logs` is guaranteed to use the block number index (that's the only WHERE condition), + // and the supplied range of blocks should be reasonably small. + // - `prune_storage_logs` is virtually guaranteed to use the primary key index since the query removes ranges w.r.t. this index. + // + // Combining these two queries or using more sophisticated queries leads to fluctuating performance due to + // unpredictable indexes being used. + let new_logs = self + .get_pks_for_latest_logs(first_l2_block_to_prune..=last_l2_block_to_prune) .await?; + let deleted_storage_logs = self.prune_storage_logs(&new_logs).await?; let deleted_l1_batches = self.delete_l1_batches(last_l1_batch_to_prune).await?; let deleted_l2_blocks = self.delete_l2_blocks(last_l2_block_to_prune).await?; @@ -194,8 +204,8 @@ impl PruningDal<'_, '_> { deleted_events, deleted_l2_to_l1_logs, deleted_call_traces, - deleted_storage_logs_from_past_batches, - deleted_storage_logs_from_pruned_batches, + overwriting_logs: new_logs.len() as u64, + deleted_storage_logs, } } else { HardPruningStats::default() @@ -314,65 +324,62 @@ impl PruningDal<'_, '_> { Ok(execution_result.rows_affected()) } - async fn prune_storage_logs_from_past_l2_blocks( + /// Gets primary keys for all latest logs in the specified L2 block range. + async fn get_pks_for_latest_logs( &mut self, l2_blocks_to_prune: ops::RangeInclusive, - ) -> DalResult { - let execution_result = sqlx::query!( + ) -> DalResult> { + sqlx::query_as!( + StorageLogPrimaryKey, r#" - DELETE FROM storage_logs + SELECT DISTINCT + ON (hashed_key) hashed_key, + miniblock_number, + operation_number + FROM + storage_logs WHERE - storage_logs.miniblock_number < $1 - AND hashed_key IN ( - SELECT - hashed_key - FROM - storage_logs - WHERE - miniblock_number BETWEEN $1 AND $2 - ) + miniblock_number BETWEEN $1 AND $2 + ORDER BY + hashed_key, + miniblock_number DESC, + operation_number DESC "#, i64::from(l2_blocks_to_prune.start().0), i64::from(l2_blocks_to_prune.end().0) ) - .instrument("hard_prune_batches_range#prune_storage_logs_from_past_l2_blocks") + .instrument("hard_prune_batches_range#get_latest_logs") .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) .report_latency() - .execute(self.storage) - .await?; - Ok(execution_result.rows_affected()) + .fetch_all(self.storage) + .await } - async fn prune_storage_logs_in_range( - &mut self, - l2_blocks_to_prune: ops::RangeInclusive, - ) -> DalResult { + /// Removes storage logs overwritten by the specified new logs. + async fn prune_storage_logs(&mut self, new_logs: &[StorageLogPrimaryKey]) -> DalResult { + let (hashed_keys, block_numbers, operation_numbers): (Vec<_>, Vec<_>, Vec<_>) = new_logs + .iter() + .map(|log| { + ( + log.hashed_key.as_slice(), + log.miniblock_number, + log.operation_number, + ) + }) + .multiunzip(); let execution_result = sqlx::query!( r#" - DELETE FROM storage_logs USING ( - SELECT - hashed_key, - MAX(ARRAY[miniblock_number, operation_number]::INT[]) AS op - FROM - storage_logs - WHERE - miniblock_number BETWEEN $1 AND $2 - GROUP BY - hashed_key - ) AS last_storage_logs + DELETE FROM storage_logs USING UNNEST($1::bytea[], $2::BIGINT[], $3::INT[]) AS new_logs (hashed_key, miniblock_number, operation_number) WHERE - storage_logs.miniblock_number BETWEEN $1 AND $2 - AND last_storage_logs.hashed_key = storage_logs.hashed_key - AND ( - storage_logs.miniblock_number != last_storage_logs.op[1] - OR storage_logs.operation_number != last_storage_logs.op[2] - ) + storage_logs.hashed_key = new_logs.hashed_key + AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number) "#, - i64::from(l2_blocks_to_prune.start().0), - i64::from(l2_blocks_to_prune.end().0) + &hashed_keys as &[&[u8]], + &block_numbers, + &operation_numbers ) - .instrument("hard_prune_batches_range#prune_storage_logs_in_range") - .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) + .instrument("hard_prune_batches_range#prune_storage_logs") + .with_arg("new_logs.len", &new_logs.len()) .report_latency() .execute(self.storage) .await?; diff --git a/core/lib/dal/src/pruning_dal/tests.rs b/core/lib/dal/src/pruning_dal/tests.rs index 4b2c6befcfaa..2670fe550c56 100644 --- a/core/lib/dal/src/pruning_dal/tests.rs +++ b/core/lib/dal/src/pruning_dal/tests.rs @@ -377,8 +377,7 @@ async fn storage_logs_pruning_works_correctly() { &[random_storage_log(2, 3), random_storage_log(3, 4)], ); assert_l2_block_storage_logs_equal(L2BlockNumber(1), &actual_logs, &[random_storage_log(1, 1)]); - assert_eq!(stats.deleted_storage_logs_from_past_batches, 0); - assert_eq!(stats.deleted_storage_logs_from_pruned_batches, 1); + assert_eq!(stats.deleted_storage_logs, 1); let stats = transaction .pruning_dal() @@ -402,8 +401,7 @@ async fn storage_logs_pruning_works_correctly() { &actual_logs, &[random_storage_log(5, 7)], ); - assert_eq!(stats.deleted_storage_logs_from_past_batches, 1); - assert_eq!(stats.deleted_storage_logs_from_pruned_batches, 1); + assert_eq!(stats.deleted_storage_logs, 2); } #[tokio::test] diff --git a/core/node/db_pruner/src/metrics.rs b/core/node/db_pruner/src/metrics.rs index 73bcefd041dd..1070ad842703 100644 --- a/core/node/db_pruner/src/metrics.rs +++ b/core/node/db_pruner/src/metrics.rs @@ -15,8 +15,8 @@ pub(super) enum MetricPruneType { enum PrunedEntityType { L1Batch, L2Block, - StorageLogFromPrunedBatch, - StorageLogFromPastBatch, + StorageLog, + OverwritingLog, // not really removed; just used to measure query complexity Event, L2ToL1Log, CallTrace, @@ -44,27 +44,22 @@ impl DbPrunerMetrics { let HardPruningStats { deleted_l1_batches, deleted_l2_blocks, - deleted_storage_logs_from_past_batches, - deleted_storage_logs_from_pruned_batches, + overwriting_logs, + deleted_storage_logs, deleted_events, deleted_call_traces, deleted_l2_to_l1_logs, } = stats; - let deleted_storage_logs = - deleted_storage_logs_from_past_batches + deleted_storage_logs_from_pruned_batches; tracing::info!( "Performed pruning of database, deleted {deleted_l1_batches} L1 batches, {deleted_l2_blocks} L2 blocks, \ - {deleted_storage_logs} storage logs ({deleted_storage_logs_from_pruned_batches} from pruned batches + \ - {deleted_storage_logs_from_past_batches} from past batches), \ + {deleted_storage_logs} storage logs ({overwriting_logs} overwriting logs), \ {deleted_events} events, {deleted_call_traces} call traces, {deleted_l2_to_l1_logs} L2-to-L1 logs" ); self.deleted_entities[&PrunedEntityType::L1Batch].observe(deleted_l1_batches); self.deleted_entities[&PrunedEntityType::L2Block].observe(deleted_l2_blocks); - self.deleted_entities[&PrunedEntityType::StorageLogFromPastBatch] - .observe(deleted_storage_logs_from_past_batches); - self.deleted_entities[&PrunedEntityType::StorageLogFromPrunedBatch] - .observe(deleted_storage_logs_from_pruned_batches); + self.deleted_entities[&PrunedEntityType::OverwritingLog].observe(overwriting_logs); + self.deleted_entities[&PrunedEntityType::StorageLog].observe(deleted_storage_logs); self.deleted_entities[&PrunedEntityType::Event].observe(deleted_events); self.deleted_entities[&PrunedEntityType::L2ToL1Log].observe(deleted_l2_to_l1_logs); self.deleted_entities[&PrunedEntityType::CallTrace].observe(deleted_call_traces); From 0b4104dbb996ec6333619ea05f3a99e6d4f3b8fa Mon Sep 17 00:00:00 2001 From: Maciej Zygmunt Date: Wed, 19 Jun 2024 14:25:59 +0200 Subject: [PATCH 21/43] feat: change `zkSync` occurences to `ZKsync` (#2227) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ This PR changes `zkSync` occurences to `ZKsync` ## Why ❔ `ZKsync` is new valid way of writing the name. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --------- Co-authored-by: Fedor Sakharov --- CONTRIBUTING.md | 2 +- README.md | 10 ++-- checks-config/era.dic | 6 +-- core/bin/contract-verifier/src/main.rs | 2 +- core/bin/external_node/README.md | 2 +- core/bin/external_node/src/config/mod.rs | 2 +- core/bin/external_node/src/main.rs | 2 +- core/bin/zksync_server/src/main.rs | 2 +- core/lib/basic_types/src/lib.rs | 12 ++--- core/lib/basic_types/src/network.rs | 4 +- core/lib/config/src/configs/chain.rs | 10 ++-- core/lib/crypto/README.md | 6 +-- .../src/eip712_signature/typed_structure.rs | 2 +- core/lib/dal/src/lib.rs | 2 +- core/lib/eth_client/src/lib.rs | 2 +- core/lib/l1_contract_interface/src/lib.rs | 2 +- core/lib/state/src/lib.rs | 2 +- core/lib/types/src/api/mod.rs | 2 +- core/lib/types/src/block.rs | 2 +- core/lib/types/src/commitment/mod.rs | 2 +- core/lib/types/src/fee.rs | 4 +- core/lib/types/src/fee_model.rs | 4 +- core/lib/types/src/l1/mod.rs | 4 +- core/lib/types/src/l2/mod.rs | 2 +- core/lib/types/src/lib.rs | 6 +-- core/lib/types/src/protocol_upgrade.rs | 2 +- core/lib/types/src/tokens.rs | 2 +- core/lib/types/src/tx/mod.rs | 2 +- core/lib/utils/src/lib.rs | 2 +- core/lib/web3_decl/src/client/network.rs | 2 +- core/lib/web3_decl/src/error.rs | 2 +- core/lib/web3_decl/src/lib.rs | 2 +- core/lib/web3_decl/src/types.rs | 4 +- core/lib/zksync_core_leftovers/src/lib.rs | 2 +- core/node/api_server/src/tx_sender/mod.rs | 2 +- .../api_server/src/web3/namespaces/eth.rs | 6 +-- core/node/block_reverter/README.md | 4 +- core/node/commitment_generator/README.md | 4 +- core/node/consensus/src/era.rs | 2 +- core/node/consistency_checker/src/lib.rs | 6 +-- core/node/eth_sender/src/tests.rs | 2 +- core/node/eth_watch/README.md | 4 +- core/node/eth_watch/src/client.rs | 2 +- .../event_processors/governance_upgrades.rs | 2 +- core/node/eth_watch/src/lib.rs | 2 +- core/node/genesis/src/lib.rs | 2 +- core/node/house_keeper/README.md | 2 +- core/node/proof_data_handler/README.md | 2 +- core/node/shared_metrics/README.md | 4 +- .../state_keeper/src/seal_criteria/mod.rs | 2 +- core/tests/loadnext/README.md | 6 +-- core/tests/loadnext/src/account/mod.rs | 2 +- core/tests/loadnext/src/account_pool.rs | 6 +-- core/tests/loadnext/src/command/api.rs | 2 +- core/tests/loadnext/src/command/tx_command.rs | 2 +- core/tests/loadnext/src/config.rs | 2 +- core/tests/loadnext/src/corrupted_tx.rs | 2 +- core/tests/loadnext/src/executor.rs | 4 +- core/tests/loadnext/src/main.rs | 4 +- core/tests/loadnext/src/sdk/abi/update-abi.sh | 2 +- core/tests/loadnext/src/sdk/ethereum/mod.rs | 6 +-- core/tests/ts-integration/README.md | 4 +- core/tests/ts-integration/contracts/README.md | 2 +- .../custom-account/SystemContractsCaller.sol | 2 +- .../custom-account/TransactionHelper.sol | 14 ++--- .../contracts/custom-account/Utils.sol | 2 +- core/tests/ts-integration/src/helpers.ts | 2 +- .../src/matchers/transaction.ts | 2 +- .../tests/ts-integration/src/prerequisites.ts | 6 +-- core/tests/ts-integration/src/types.ts | 6 +-- .../ts-integration/tests/api/web3.test.ts | 4 +- .../ts-integration/tests/contracts.test.ts | 2 +- core/tests/ts-integration/tests/l1.test.ts | 2 +- .../ts-integration/tests/mempool.test.ts | 2 +- .../ts-integration/tests/self-unit.test.ts | 2 +- docs/guides/advanced/01_initialization.md | 10 ++-- docs/guides/advanced/02_deposits.md | 4 +- docs/guides/advanced/03_withdrawals.md | 2 +- .../guides/advanced/0_alternative_vm_intro.md | 16 +++--- docs/guides/advanced/contracts.md | 26 ++++----- docs/guides/advanced/fee_model.md | 6 +-- .../guides/advanced/how_l2_messaging_works.md | 2 +- docs/guides/advanced/how_transaction_works.md | 2 +- docs/guides/advanced/pubdata-with-blobs.md | 4 +- docs/guides/advanced/pubdata.md | 4 +- docs/guides/architecture.md | 54 +++++++++---------- docs/guides/development.md | 2 +- docs/guides/external-node/00_quick_start.md | 6 +-- docs/guides/external-node/01_intro.md | 28 +++++----- docs/guides/external-node/02_configuration.md | 18 +++---- docs/guides/external-node/03_running.md | 16 +++--- docs/guides/external-node/04_observability.md | 14 ++--- .../external-node/05_troubleshooting.md | 10 ++-- docs/guides/external-node/06_components.md | 34 ++++++------ .../prepared_configs/mainnet-config.env | 2 +- .../testnet-goerli-config-deprecated.env | 2 +- .../testnet-sepolia-config.env | 2 +- docs/guides/launch.md | 2 +- docs/guides/repositories.md | 22 ++++---- docs/guides/setup-dev.md | 4 +- docs/specs/blocks_batches.md | 2 +- docs/specs/data_availability/pubdata.md | 14 ++--- docs/specs/l1_l2_communication/l1_to_l2.md | 4 +- docs/specs/l1_smart_contracts.md | 10 ++-- docs/specs/prover/overview.md | 8 +-- docs/specs/prover/zk_terminology.md | 10 ++-- docs/specs/zk_evm/account_abstraction.md | 2 +- docs/specs/zk_evm/bootloader.md | 12 ++--- docs/specs/zk_evm/fee_model.md | 32 +++++------ docs/specs/zk_evm/precompiles.md | 4 +- docs/specs/zk_evm/system_contracts.md | 24 ++++----- .../compiler/instructions/evm/call.md | 2 +- .../compiler/instructions/evm/create.md | 2 +- .../compiler/instructions/evm/return.md | 2 +- .../compiler/instructions/evmla.md | 6 +-- .../compiler/instructions/extensions/call.md | 2 +- .../instructions/extensions/overview.md | 4 +- .../instructions/extensions/verbatim.md | 2 +- .../compiler/instructions/overview.md | 10 ++-- .../compiler/instructions/yul.md | 6 +-- .../vm_specification/compiler/overview.md | 10 ++-- .../compiler/system_contracts.md | 8 +-- etc/contracts-test-data/README.md | 2 +- .../custom-account/SystemContractsCaller.sol | 2 +- .../custom-account/TransactionHelper.sol | 14 ++--- .../contracts/custom-account/Utils.sol | 2 +- etc/env/base/README.md | 4 +- etc/env/base/chain.toml | 12 ++--- etc/env/base/contracts.toml | 2 +- etc/env/base/private.toml | 2 +- etc/test_config/README.md | 4 +- flake.nix | 2 +- .../local-setup-preparation/README.md | 2 +- .../local-setup-preparation/src/index.ts | 2 +- infrastructure/protocol-upgrade/src/index.ts | 2 +- infrastructure/zk/src/index.ts | 2 +- prover/prover_cli/README.md | 2 +- zk_toolbox/Cargo.toml | 2 +- zk_toolbox/crates/common/src/term/logger.rs | 2 +- zk_toolbox/crates/config/src/consts.rs | 2 +- 140 files changed, 392 insertions(+), 392 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89789b081502..2676289d0f3a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ We aim to make it as easy as possible to contribute to the mission. This is stil and suggestions here too. Some resources to help: 1. [In-repo docs aimed at developers](docs) -2. [zkSync Era docs!](https://docs.zksync.io) +2. [ZKsync Era docs!](https://docs.zksync.io) 3. Company links can be found in the [repositories' readme](README.md) ## Code of Conduct diff --git a/README.md b/README.md index 4700b1b43a9e..013d932aa1ae 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ -# zkSync Era: A ZK Rollup For Scaling Ethereum +# ZKsync Era: A ZK Rollup For Scaling Ethereum [![Logo](eraLogo.png)](https://zksync.io/) -zkSync Era is a layer 2 rollup that uses zero-knowledge proofs to scale Ethereum without compromising on security or +ZKsync Era is a layer 2 rollup that uses zero-knowledge proofs to scale Ethereum without compromising on security or decentralization. Since it's EVM compatible (Solidity/Vyper), 99% of Ethereum projects can redeploy without refactoring -or re-auditing a single line of code. zkSync Era also uses an LLVM-based compiler that will eventually let developers +or re-auditing a single line of code. ZKsync Era also uses an LLVM-based compiler that will eventually let developers write smart contracts in C++, Rust and other popular languages. ## Knowledge Index @@ -27,7 +27,7 @@ The following questions will be answered by the following resources: ## License -zkSync Era is distributed under the terms of either +ZKsync Era is distributed under the terms of either - Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or ) - MIT license ([LICENSE-MIT](LICENSE-MIT) or ) @@ -47,7 +47,7 @@ at your option. ## Disclaimer -zkSync Era has been through lots of testing and audits. Although it is live, it is still in alpha state and will go +ZKsync Era has been through lots of testing and audits. Although it is live, it is still in alpha state and will go through more audits and bug bounty programs. We would love to hear our community's thoughts and suggestions about it! It is important to state that forking it now can potentially lead to missing important security updates, critical features, and performance improvements. diff --git a/checks-config/era.dic b/checks-config/era.dic index 0b55a55c83ea..3f4c8fc8fa4f 100644 --- a/checks-config/era.dic +++ b/checks-config/era.dic @@ -135,7 +135,7 @@ boolean prover timestamp H160 -zkSync +ZKsync AccessList miniblock member₁ @@ -212,7 +212,7 @@ EOAs zeroized cardinality -// zkSync-related words +// ZKsync-related words matterlabs zkweb zksync @@ -610,7 +610,7 @@ DBs unexecutable RLP DAL -zkSync's +ZKsync's l2_to_l1 PoW coinbase diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 5789422641c8..118e7f41be97 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -112,7 +112,7 @@ async fn update_compiler_versions(connection_pool: &ConnectionPool) { use zksync_config::configs::DatabaseSecrets; #[derive(StructOpt)] -#[structopt(name = "zkSync contract code verifier", author = "Matter Labs")] +#[structopt(name = "ZKsync contract code verifier", author = "Matter Labs")] struct Opt { /// Number of jobs to process. If None, runs indefinitely. #[structopt(long)] diff --git a/core/bin/external_node/README.md b/core/bin/external_node/README.md index d6fa78dbd3d3..335ceed7b719 100644 --- a/core/bin/external_node/README.md +++ b/core/bin/external_node/README.md @@ -1,4 +1,4 @@ -# zkSync External Node +# ZKsync External Node This application is a read replica that can sync from the main node and serve the state locally. diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index e329150721c0..9cd6a758a25c 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -647,7 +647,7 @@ pub(crate) struct RequiredENConfig { /// L1 chain ID (e.g., 9 for Ethereum mainnet). This ID will be checked against the `eth_client_url` RPC provider on initialization /// to ensure that there's no mismatch between the expected and actual L1 network. pub l1_chain_id: L1ChainId, - /// L2 chain ID (e.g., 270 for zkSync Era mainnet). This ID will be checked against the `main_node_url` RPC provider on initialization + /// L2 chain ID (e.g., 270 for ZKsync Era mainnet). This ID will be checked against the `main_node_url` RPC provider on initialization /// to ensure that there's no mismatch between the expected and actual L2 network. pub l2_chain_id: L2ChainId, diff --git a/core/bin/external_node/src/main.rs b/core/bin/external_node/src/main.rs index 04435f66bf4b..0adf3ddf8cb5 100644 --- a/core/bin/external_node/src/main.rs +++ b/core/bin/external_node/src/main.rs @@ -692,7 +692,7 @@ async fn shutdown_components( Ok(()) } -/// External node for zkSync Era. +/// External node for ZKsync Era. #[derive(Debug, Parser)] #[command(author = "Matter Labs", version)] struct Cli { diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index c51cc5380253..1c54895863c0 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -38,7 +38,7 @@ mod node_builder; #[global_allocator] static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[derive(Debug, Parser)] -#[command(author = "Matter Labs", version, about = "zkSync operator node", long_about = None)] +#[command(author = "Matter Labs", version, about = "ZKsync operator node", long_about = None)] struct Cli { /// Generate genesis block for the first contract deployment using temporary DB. #[arg(long)] diff --git a/core/lib/basic_types/src/lib.rs b/core/lib/basic_types/src/lib.rs index 5c54b0a21696..a55705886c55 100644 --- a/core/lib/basic_types/src/lib.rs +++ b/core/lib/basic_types/src/lib.rs @@ -1,4 +1,4 @@ -//! The declaration of the most primitive types used in zkSync network. +//! The declaration of the most primitive types used in ZKsync network. //! //! Most of them are just re-exported from the `web3` crate. @@ -86,7 +86,7 @@ impl TryFrom for AccountTreeId { } } -/// ChainId in the zkSync network. +/// ChainId in the ZKsync network. #[derive(Copy, Clone, Debug, Serialize, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct L2ChainId(u64); @@ -183,13 +183,13 @@ impl From for L2ChainId { } basic_type!( - /// zkSync network block sequential index. + /// ZKsync network block sequential index. L2BlockNumber, u32 ); basic_type!( - /// zkSync L1 batch sequential index. + /// ZKsync L1 batch sequential index. L1BatchNumber, u32 ); @@ -201,13 +201,13 @@ basic_type!( ); basic_type!( - /// zkSync account nonce. + /// ZKsync account nonce. Nonce, u32 ); basic_type!( - /// Unique identifier of the priority operation in the zkSync network. + /// Unique identifier of the priority operation in the ZKsync network. PriorityOpId, u64 ); diff --git a/core/lib/basic_types/src/network.rs b/core/lib/basic_types/src/network.rs index 5f4683aeb672..cfa82e8c8464 100644 --- a/core/lib/basic_types/src/network.rs +++ b/core/lib/basic_types/src/network.rs @@ -1,4 +1,4 @@ -//! The network where the zkSync resides. +//! The network where the ZKsync resides. //! // Built-in uses @@ -12,7 +12,7 @@ use crate::L1ChainId; // Local uses -/// Network to be used for a zkSync client. +/// Network to be used for a ZKsync client. #[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub enum Network { diff --git a/core/lib/config/src/configs/chain.rs b/core/lib/config/src/configs/chain.rs index ade0f9d4226f..c1abd1fea102 100644 --- a/core/lib/config/src/configs/chain.rs +++ b/core/lib/config/src/configs/chain.rs @@ -9,11 +9,11 @@ use zksync_basic_types::{ pub struct NetworkConfig { /// Name of the used Ethereum network, e.g. `localhost` or `rinkeby`. pub network: Network, - /// Name of current zkSync network + /// Name of current ZKsync network /// Used for Sentry environment pub zksync_network: String, - /// ID of current zkSync network treated as ETH network ID. - /// Used to distinguish zkSync from other Web3-capable networks. + /// ID of current ZKsync network treated as ETH network ID. + /// Used to distinguish ZKsync from other Web3-capable networks. pub zksync_network_id: L2ChainId, } @@ -29,10 +29,10 @@ impl NetworkConfig { } /// An enum that represents the version of the fee model to use. -/// - `V1`, the first model that was used in zkSync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. +/// - `V1`, the first model that was used in ZKsync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. /// Also, the fair L2 gas price is expected to only include the proving/computation price for the operator and not the costs that come from /// processing the batch on L1. -/// - `V2`, the second model that was used in zkSync Era. There the pubdata price might be independent from the L1 gas price. Also, +/// - `V2`, the second model that was used in ZKsync Era. There the pubdata price might be independent from the L1 gas price. Also, /// The fair L2 gas price is expected to both the proving/computation price for the operator and the costs that come from /// processing the batch on L1. #[derive(Debug, Clone, Copy, Deserialize, PartialEq, Eq)] diff --git a/core/lib/crypto/README.md b/core/lib/crypto/README.md index e224b2732d34..38b5a306a9bd 100644 --- a/core/lib/crypto/README.md +++ b/core/lib/crypto/README.md @@ -1,10 +1,10 @@ -# zkSync crypto. Essential cryptography primitives for the zkSync network +# ZKsync crypto. Essential cryptography primitives for the ZKsync network -`zksync_crypto` is a crate containing essential zkSync cryptographic primitives, such as private keys and hashers. +`zksync_crypto` is a crate containing essential ZKsync cryptographic primitives, such as private keys and hashers. ## License -`zksync_crypto` is a part of zkSync stack, which is distributed under the terms of both the MIT license and the Apache +`zksync_crypto` is a part of ZKsync stack, which is distributed under the terms of both the MIT license and the Apache License (Version 2.0). See [LICENSE-APACHE](../../../LICENSE-APACHE), [LICENSE-MIT](../../../LICENSE-MIT) for details. diff --git a/core/lib/crypto_primitives/src/eip712_signature/typed_structure.rs b/core/lib/crypto_primitives/src/eip712_signature/typed_structure.rs index 1315ccb06a26..a08273c0a037 100644 --- a/core/lib/crypto_primitives/src/eip712_signature/typed_structure.rs +++ b/core/lib/crypto_primitives/src/eip712_signature/typed_structure.rs @@ -160,7 +160,7 @@ impl Eip712Domain { pub const NAME: &'static str = "zkSync"; /// Version of the protocol. While there may be `2.x` releases, the minor release version bump /// should not be breaking, meaning that clients from the `2.x-1` version should be able to communicate - /// with zkSync server. Thus `VERSION` corresponds to the major version only. + /// with ZKsync server. Thus `VERSION` corresponds to the major version only. pub const VERSION: &'static str = "2"; pub fn new(chain_id: L2ChainId) -> Self { diff --git a/core/lib/dal/src/lib.rs b/core/lib/dal/src/lib.rs index 45d1f94b486d..0a2ed3bdd641 100644 --- a/core/lib/dal/src/lib.rs +++ b/core/lib/dal/src/lib.rs @@ -1,4 +1,4 @@ -//! Data access layer (DAL) for zkSync Era. +//! Data access layer (DAL) for ZKsync Era. // Linter settings. #![warn(clippy::cast_lossless)] diff --git a/core/lib/eth_client/src/lib.rs b/core/lib/eth_client/src/lib.rs index 2adac587b66c..6e24047dd48c 100644 --- a/core/lib/eth_client/src/lib.rs +++ b/core/lib/eth_client/src/lib.rs @@ -152,7 +152,7 @@ pub trait EthInterface: Sync + Send { /// /// The example use cases for this trait would be: /// -/// - An operator that sends transactions and interacts with zkSync contract. +/// - An operator that sends transactions and interacts with ZKsync contract. /// - A wallet implementation in the SDK that is tied to a user's account. /// /// When adding a method to this trait: diff --git a/core/lib/l1_contract_interface/src/lib.rs b/core/lib/l1_contract_interface/src/lib.rs index fc96347bf70b..26a9aefa9f1e 100644 --- a/core/lib/l1_contract_interface/src/lib.rs +++ b/core/lib/l1_contract_interface/src/lib.rs @@ -1,4 +1,4 @@ -//! Utilities for interacting with the zkSync L1 contract +//! Utilities for interacting with the ZKsync L1 contract //! //! Provides utilities both to encode input data for the contract and to decode //! the data provided by the contract. diff --git a/core/lib/state/src/lib.rs b/core/lib/state/src/lib.rs index 1359e62824f5..b01d4fd35375 100644 --- a/core/lib/state/src/lib.rs +++ b/core/lib/state/src/lib.rs @@ -1,4 +1,4 @@ -//! Execution of transaction in zkSync Era +//! Execution of transaction in ZKsync Era // Linter settings. #![warn(missing_debug_implementations, missing_docs, bare_trait_objects)] diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 6e22e17de673..ce21a754c7aa 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -89,7 +89,7 @@ impl<'de> Deserialize<'de> for BlockNumber { } } -/// Block unified identifier in terms of zkSync +/// Block unified identifier in terms of ZKsync /// /// This is an utility structure that cannot be (de)serialized, it has to be created manually. /// The reason is because Web3 API provides multiple methods for referring block either by hash or number, diff --git a/core/lib/types/src/block.rs b/core/lib/types/src/block.rs index c9b1c528f7e3..221b9b4d63ff 100644 --- a/core/lib/types/src/block.rs +++ b/core/lib/types/src/block.rs @@ -200,7 +200,7 @@ pub struct L2BlockHasher { } impl L2BlockHasher { - /// At the beginning of the zkSync, the hashes of the blocks could be calculated as the hash of their number. + /// At the beginning of the ZKsync, the hashes of the blocks could be calculated as the hash of their number. /// This method returns the hash of such L2 blocks. pub fn legacy_hash(l2_block_number: L2BlockNumber) -> H256 { H256(keccak256(&l2_block_number.0.to_be_bytes())) diff --git a/core/lib/types/src/commitment/mod.rs b/core/lib/types/src/commitment/mod.rs index 7c4184e5e189..61c2d7b5ea27 100644 --- a/core/lib/types/src/commitment/mod.rs +++ b/core/lib/types/src/commitment/mod.rs @@ -1,7 +1,7 @@ //! Data structures that have more metadata than their primary versions declared in this crate. //! For example, L1 batch defined here has the `root_hash` field which is absent in `L1BatchHeader`. //! -//! Existence of this module is caused by the execution model of zkSync: when executing transactions, +//! Existence of this module is caused by the execution model of ZKsync: when executing transactions, //! we aim to avoid expensive operations like the state root hash recalculation. State root hash is not //! required for the rollup to execute L1 batches, it's needed for the proof generation and the Ethereum //! transactions, thus the calculations are done separately and asynchronously. diff --git a/core/lib/types/src/fee.rs b/core/lib/types/src/fee.rs index 5f32beb2fd4c..524015cdd095 100644 --- a/core/lib/types/src/fee.rs +++ b/core/lib/types/src/fee.rs @@ -57,9 +57,9 @@ impl Default for TransactionExecutionMetrics { pub struct Fee { /// The limit of gas that are to be spent on the actual transaction. pub gas_limit: U256, - /// zkSync version of EIP1559 maxFeePerGas. + /// ZKsync version of EIP1559 maxFeePerGas. pub max_fee_per_gas: U256, - /// zkSync version of EIP1559 maxPriorityFeePerGas. + /// ZKsync version of EIP1559 maxPriorityFeePerGas. pub max_priority_fee_per_gas: U256, /// The maximal gas per pubdata byte the user agrees to. pub gas_per_pubdata_limit: U256, diff --git a/core/lib/types/src/fee_model.rs b/core/lib/types/src/fee_model.rs index 79c9a94eda9f..9c2cc4d2aaf8 100644 --- a/core/lib/types/src/fee_model.rs +++ b/core/lib/types/src/fee_model.rs @@ -157,10 +157,10 @@ pub struct PubdataIndependentBatchFeeModelInput { } /// The enum which represents the version of the fee model. It is used to determine which fee model should be used for the batch. -/// - `V1`, the first model that was used in zkSync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. +/// - `V1`, the first model that was used in ZKsync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. /// Also, the fair L2 gas price is expected to only include the proving/computation price for the operator and not the costs that come from /// processing the batch on L1. -/// - `V2`, the second model that was used in zkSync Era. There the pubdata price might be independent from the L1 gas price. Also, +/// - `V2`, the second model that was used in ZKsync Era. There the pubdata price might be independent from the L1 gas price. Also, /// The fair L2 gas price is expected to both the proving/computation price for the operator and the costs that come from /// processing the batch on L1. #[derive(Debug, Clone, Copy, Serialize, Deserialize)] diff --git a/core/lib/types/src/l1/mod.rs b/core/lib/types/src/l1/mod.rs index 348600b6ee89..05f08987a2d3 100644 --- a/core/lib/types/src/l1/mod.rs +++ b/core/lib/types/src/l1/mod.rs @@ -1,4 +1,4 @@ -//! Definition of zkSync network priority operations: operations initiated from the L1. +//! Definition of ZKsync network priority operations: operations initiated from the L1. use std::convert::TryFrom; @@ -119,7 +119,7 @@ pub struct L1TxCommonData { pub op_processing_type: OpProcessingType, /// Priority operations queue type. pub priority_queue_type: PriorityQueueType, - /// Tx hash of the transaction in the zkSync network. Calculated as the encoded transaction data hash. + /// Tx hash of the transaction in the ZKsync network. Calculated as the encoded transaction data hash. pub canonical_tx_hash: H256, /// The amount of ETH that should be minted with this transaction pub to_mint: U256, diff --git a/core/lib/types/src/l2/mod.rs b/core/lib/types/src/l2/mod.rs index 57edc6181c8a..5a5276407529 100644 --- a/core/lib/types/src/l2/mod.rs +++ b/core/lib/types/src/l2/mod.rs @@ -31,7 +31,7 @@ pub enum TransactionType { LegacyTransaction = 0, EIP2930Transaction = 1, EIP1559Transaction = 2, - // EIP 712 transaction with additional fields specified for zkSync + // EIP 712 transaction with additional fields specified for ZKsync EIP712Transaction = EIP_712_TX_TYPE as u32, PriorityOpTransaction = PRIORITY_OPERATION_L2_TX_TYPE as u32, ProtocolUpgradeTransaction = PROTOCOL_UPGRADE_TX_TYPE as u32, diff --git a/core/lib/types/src/lib.rs b/core/lib/types/src/lib.rs index 2617bf0e4984..3c3a96c297d7 100644 --- a/core/lib/types/src/lib.rs +++ b/core/lib/types/src/lib.rs @@ -1,6 +1,6 @@ -//! zkSync types: essential type definitions for zkSync network. +//! ZKsync types: essential type definitions for ZKsync network. //! -//! `zksync_types` is a crate containing essential zkSync network types, such as transactions, operations and +//! `zksync_types` is a crate containing essential ZKsync network types, such as transactions, operations and //! blockchain primitives. #![allow(clippy::upper_case_acronyms, clippy::derive_partial_eq_without_eq)] @@ -63,7 +63,7 @@ pub mod proto; pub mod transaction_request; pub mod utils; -/// Denotes the first byte of the special zkSync's EIP-712-signed transaction. +/// Denotes the first byte of the special ZKsync's EIP-712-signed transaction. pub const EIP_712_TX_TYPE: u8 = 0x71; /// Denotes the first byte of the `EIP-1559` transaction. diff --git a/core/lib/types/src/protocol_upgrade.rs b/core/lib/types/src/protocol_upgrade.rs index c1bcc2f5cace..c0d7267ebfae 100644 --- a/core/lib/types/src/protocol_upgrade.rs +++ b/core/lib/types/src/protocol_upgrade.rs @@ -352,7 +352,7 @@ pub struct ProtocolUpgradeTxCommonData { pub gas_per_pubdata_limit: U256, /// Block in which Ethereum transaction was included. pub eth_block: u64, - /// Tx hash of the transaction in the zkSync network. Calculated as the encoded transaction data hash. + /// Tx hash of the transaction in the ZKsync network. Calculated as the encoded transaction data hash. pub canonical_tx_hash: H256, /// The amount of ETH that should be minted with this transaction pub to_mint: U256, diff --git a/core/lib/types/src/tokens.rs b/core/lib/types/src/tokens.rs index 26aec479498d..ddabaffa2319 100644 --- a/core/lib/types/src/tokens.rs +++ b/core/lib/types/src/tokens.rs @@ -9,7 +9,7 @@ pub struct TokenInfo { pub metadata: TokenMetadata, } -/// Relevant information about tokens supported by zkSync protocol. +/// Relevant information about tokens supported by ZKsync protocol. #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] pub struct TokenMetadata { /// Token name (e.g. "Ethereum" or "USD Coin") diff --git a/core/lib/types/src/tx/mod.rs b/core/lib/types/src/tx/mod.rs index 9bf38aa19555..7078f4ee3fe3 100644 --- a/core/lib/types/src/tx/mod.rs +++ b/core/lib/types/src/tx/mod.rs @@ -1,6 +1,6 @@ //! `transactions` is module that holds the essential information for every transaction. //! -//! Since in zkSync Era every operation can be executed either from the contract or rollup, +//! Since in ZKsync Era every operation can be executed either from the contract or rollup, //! it makes more sense to define the contents of each transaction chain-agnostic, and extent this data //! with metadata (such as fees and/or signatures) for L1 and L2 separately. diff --git a/core/lib/utils/src/lib.rs b/core/lib/utils/src/lib.rs index 1c17d4efe264..7f9304e3110c 100644 --- a/core/lib/utils/src/lib.rs +++ b/core/lib/utils/src/lib.rs @@ -1,4 +1,4 @@ -//! Various helpers used in the zkSync stack. +//! Various helpers used in the ZKsync stack. pub mod bytecode; mod convert; diff --git a/core/lib/web3_decl/src/client/network.rs b/core/lib/web3_decl/src/client/network.rs index dabde86678bf..2e7dcce9937f 100644 --- a/core/lib/web3_decl/src/client/network.rs +++ b/core/lib/web3_decl/src/client/network.rs @@ -52,7 +52,7 @@ impl From for L2 { } } -/// Associates a type with a particular type of RPC networks, such as Ethereum or zkSync Era. RPC traits created using `jsonrpsee::rpc` +/// Associates a type with a particular type of RPC networks, such as Ethereum or ZKsync Era. RPC traits created using `jsonrpsee::rpc` /// can use `ForNetwork` as a client boundary to restrict which implementations can call their methods. pub trait ForNetwork { /// Network that the type is associated with. diff --git a/core/lib/web3_decl/src/error.rs b/core/lib/web3_decl/src/error.rs index e80ea23d8e3a..1ea737a947f5 100644 --- a/core/lib/web3_decl/src/error.rs +++ b/core/lib/web3_decl/src/error.rs @@ -1,4 +1,4 @@ -//! Definition of errors that can occur in the zkSync Web3 API. +//! Definition of errors that can occur in the ZKsync Web3 API. use std::{ collections::HashMap, diff --git a/core/lib/web3_decl/src/lib.rs b/core/lib/web3_decl/src/lib.rs index 7146a87099ce..c104668d5977 100644 --- a/core/lib/web3_decl/src/lib.rs +++ b/core/lib/web3_decl/src/lib.rs @@ -1,4 +1,4 @@ -//! `zksync_web3_decl` is a collection of common types required for zkSync Web3 API +//! `zksync_web3_decl` is a collection of common types required for ZKsync Web3 API //! and also `jsonrpsee`-based declaration of server and client traits. //! //! Web3 namespaces are declared in `namespaces` module. diff --git a/core/lib/web3_decl/src/types.rs b/core/lib/web3_decl/src/types.rs index ec6bbed46883..41902e408e7a 100644 --- a/core/lib/web3_decl/src/types.rs +++ b/core/lib/web3_decl/src/types.rs @@ -3,7 +3,7 @@ //! Most of the types are re-exported from the `web3` crate, but some of them maybe extended with //! new variants (enums) or optional fields (structures). //! -//! These "extensions" are required to provide more zkSync-specific information while remaining Web3-compilant. +//! These "extensions" are required to provide more ZKsync-specific information while remaining Web3-compilant. use core::{ convert::{TryFrom, TryInto}, @@ -21,7 +21,7 @@ pub use zksync_types::{ Address, Transaction, H160, H256, H64, U256, U64, }; -/// Token in the zkSync network +/// Token in the ZKsync network #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct Token { diff --git a/core/lib/zksync_core_leftovers/src/lib.rs b/core/lib/zksync_core_leftovers/src/lib.rs index 1ed84263c2d5..b4194f99f450 100644 --- a/core/lib/zksync_core_leftovers/src/lib.rs +++ b/core/lib/zksync_core_leftovers/src/lib.rs @@ -78,7 +78,7 @@ use zksync_web3_decl::client::{Client, DynClient, L1}; pub mod temp_config_store; -/// Inserts the initial information about zkSync tokens into the database. +/// Inserts the initial information about ZKsync tokens into the database. pub async fn genesis_init( genesis_config: GenesisConfig, database_secrets: &DatabaseSecrets, diff --git a/core/node/api_server/src/tx_sender/mod.rs b/core/node/api_server/src/tx_sender/mod.rs index 1dd3f4c6e941..a6bbbf9ffa04 100644 --- a/core/node/api_server/src/tx_sender/mod.rs +++ b/core/node/api_server/src/tx_sender/mod.rs @@ -1,4 +1,4 @@ -//! Helper module to submit transactions into the zkSync Network. +//! Helper module to submit transactions into the ZKsync Network. use std::{sync::Arc, time::Instant}; diff --git a/core/node/api_server/src/web3/namespaces/eth.rs b/core/node/api_server/src/web3/namespaces/eth.rs index d1801fde6e4d..397ce77c050f 100644 --- a/core/node/api_server/src/web3/namespaces/eth.rs +++ b/core/node/api_server/src/web3/namespaces/eth.rs @@ -840,17 +840,17 @@ impl EthNamespace { } pub fn uncle_count_impl(&self, _block: BlockId) -> Option { - // We don't have uncles in zkSync. + // We don't have uncles in ZKsync. Some(0.into()) } pub fn hashrate_impl(&self) -> U256 { - // zkSync is not a PoW chain. + // ZKsync is not a PoW chain. U256::zero() } pub fn mining_impl(&self) -> bool { - // zkSync is not a PoW chain. + // ZKsync is not a PoW chain. false } diff --git a/core/node/block_reverter/README.md b/core/node/block_reverter/README.md index 9d82fb0d189d..0c696dca4761 100644 --- a/core/node/block_reverter/README.md +++ b/core/node/block_reverter/README.md @@ -1,4 +1,4 @@ -# zkSync Era Block reverter +# ZKsync Era Block reverter -This crate contains functionality for rolling back state of a zkSync Era node and reverting committed L1 batches on +This crate contains functionality for rolling back state of a ZKsync Era node and reverting committed L1 batches on Ethereum. diff --git a/core/node/commitment_generator/README.md b/core/node/commitment_generator/README.md index da99ca9403a1..eaa5017e8f0d 100644 --- a/core/node/commitment_generator/README.md +++ b/core/node/commitment_generator/README.md @@ -1,4 +1,4 @@ -# zkSync Era commitment generator +# ZKsync Era commitment generator -This crate contains an implementation of the zkSync Era commitment generator component, which is responsible for the +This crate contains an implementation of the ZKsync Era commitment generator component, which is responsible for the calculation commitment info for L1 batches. diff --git a/core/node/consensus/src/era.rs b/core/node/consensus/src/era.rs index a8477a8bb672..0e73c29f7741 100644 --- a/core/node/consensus/src/era.rs +++ b/core/node/consensus/src/era.rs @@ -1,5 +1,5 @@ //! This module provides convenience functions to run consensus components in different modes -//! as expected by the zkSync Era. +//! as expected by the ZKsync Era. //! //! This module simply glues APIs that are already publicly exposed by the `consensus` module, //! so in case any custom behavior is needed, these APIs should be used directly. diff --git a/core/node/consistency_checker/src/lib.rs b/core/node/consistency_checker/src/lib.rs index 79ce137560c6..e4634c86e403 100644 --- a/core/node/consistency_checker/src/lib.rs +++ b/core/node/consistency_checker/src/lib.rs @@ -302,9 +302,9 @@ pub fn detect_da( #[derive(Debug)] pub struct ConsistencyChecker { - /// ABI of the zkSync contract + /// ABI of the ZKsync contract contract: ethabi::Contract, - /// Address of the zkSync diamond proxy on L1 + /// Address of the ZKsync diamond proxy on L1 diamond_proxy_addr: Option
, /// How many past batches to check when starting max_batches_to_recheck: u32, @@ -382,7 +382,7 @@ impl ConsistencyChecker { let event = self .contract .event("BlockCommit") - .context("`BlockCommit` event not found for zkSync L1 contract") + .context("`BlockCommit` event not found for ZKsync L1 contract") .map_err(CheckError::Internal)?; let committed_batch_numbers_by_logs = diff --git a/core/node/eth_sender/src/tests.rs b/core/node/eth_sender/src/tests.rs index 00b02c2fe9b5..a3bb9951f44a 100644 --- a/core/node/eth_sender/src/tests.rs +++ b/core/node/eth_sender/src/tests.rs @@ -178,7 +178,7 @@ impl EthSenderTester { commitment_mode, ), gateway.clone(), - // zkSync contract address + // ZKsync contract address Address::random(), contracts_config.l1_multicall3_addr, Address::random(), diff --git a/core/node/eth_watch/README.md b/core/node/eth_watch/README.md index f805f3e4c389..5b4dd5c2ea0d 100644 --- a/core/node/eth_watch/README.md +++ b/core/node/eth_watch/README.md @@ -1,6 +1,6 @@ -# zkSync Era Eth Watcher +# ZKsync Era Eth Watcher -This crate contains an implementation of the zkSync Era Eth Watcher component, which fetches the changes from the +This crate contains an implementation of the ZKsync Era Eth Watcher component, which fetches the changes from the corresponding L1 contract. ## Overview diff --git a/core/node/eth_watch/src/client.rs b/core/node/eth_watch/src/client.rs index 604ea2f471cc..764573002996 100644 --- a/core/node/eth_watch/src/client.rs +++ b/core/node/eth_watch/src/client.rs @@ -57,7 +57,7 @@ impl EthHttpQueryClient { confirmations_for_eth_event: Option, ) -> Self { tracing::debug!( - "New eth client, zkSync addr: {:x}, governance addr: {:?}", + "New eth client, ZKsync addr: {:x}, governance addr: {:?}", diamond_proxy_addr, governance_address ); diff --git a/core/node/eth_watch/src/event_processors/governance_upgrades.rs b/core/node/eth_watch/src/event_processors/governance_upgrades.rs index d26cfe6dbd9b..72f5c411892f 100644 --- a/core/node/eth_watch/src/event_processors/governance_upgrades.rs +++ b/core/node/eth_watch/src/event_processors/governance_upgrades.rs @@ -14,7 +14,7 @@ use crate::{ /// Listens to operation events coming from the governance contract and saves new protocol upgrade proposals to the database. #[derive(Debug)] pub struct GovernanceUpgradesEventProcessor { - // zkSync diamond proxy + // ZKsync diamond proxy target_contract_address: Address, /// Last protocol version seen. Used to skip events for already known upgrade proposals. last_seen_protocol_version: ProtocolSemanticVersion, diff --git a/core/node/eth_watch/src/lib.rs b/core/node/eth_watch/src/lib.rs index 7cb0064c3d73..7c27a6322c2f 100644 --- a/core/node/eth_watch/src/lib.rs +++ b/core/node/eth_watch/src/lib.rs @@ -1,6 +1,6 @@ //! Ethereum watcher polls the Ethereum node for the relevant events, such as priority operations (aka L1 transactions), //! protocol upgrades etc. -//! New events are accepted to the zkSync network once they have the sufficient amount of L1 confirmations. +//! New events are accepted to the ZKsync network once they have the sufficient amount of L1 confirmations. use std::time::Duration; diff --git a/core/node/genesis/src/lib.rs b/core/node/genesis/src/lib.rs index bfa6b77cbfef..461f208e3012 100644 --- a/core/node/genesis/src/lib.rs +++ b/core/node/genesis/src/lib.rs @@ -1,4 +1,4 @@ -//! This module aims to provide a genesis setup for the zkSync Era network. +//! This module aims to provide a genesis setup for the ZKsync Era network. //! It initializes the Merkle tree with the basic setup (such as fields of special service accounts), //! setups the required databases, and outputs the data required to initialize a smart contract. diff --git a/core/node/house_keeper/README.md b/core/node/house_keeper/README.md index 4f8c399a85b2..eaeb7c14b203 100644 --- a/core/node/house_keeper/README.md +++ b/core/node/house_keeper/README.md @@ -1,4 +1,4 @@ -# zkSync Era housekeeper +# ZKsync Era housekeeper This crate contains functionality for performing “administrative” work to keep the system flowing. It does: diff --git a/core/node/proof_data_handler/README.md b/core/node/proof_data_handler/README.md index 8c3392c5b1fe..8cc48fe0aa35 100644 --- a/core/node/proof_data_handler/README.md +++ b/core/node/proof_data_handler/README.md @@ -1,3 +1,3 @@ -# zkSync Era Proof data handler +# ZKsync Era Proof data handler This crate contains functionality for sending proof-related info from `Server` to `Prover` and back. diff --git a/core/node/shared_metrics/README.md b/core/node/shared_metrics/README.md index 45aa229f808a..e60cf9176369 100644 --- a/core/node/shared_metrics/README.md +++ b/core/node/shared_metrics/README.md @@ -1,3 +1,3 @@ -# zkSync Era shared metrics +# ZKsync Era shared metrics -This crate contains the definitions of various metrics that are shared among different zkSync Era components. +This crate contains the definitions of various metrics that are shared among different ZKsync Era components. diff --git a/core/node/state_keeper/src/seal_criteria/mod.rs b/core/node/state_keeper/src/seal_criteria/mod.rs index 505d9944149d..a721c53b6467 100644 --- a/core/node/state_keeper/src/seal_criteria/mod.rs +++ b/core/node/state_keeper/src/seal_criteria/mod.rs @@ -122,7 +122,7 @@ pub enum SealResolution { /// tx in the next block. /// While it may be kinda counter-intuitive that we first execute transaction and just then /// decided whether we should include it into the block or not, it is required by the architecture of - /// zkSync Era. We may not know, for example, how much gas block will consume, because 1) smart contract + /// ZKsync Era. We may not know, for example, how much gas block will consume, because 1) smart contract /// execution is hard to predict and 2) we may have writes to the same storage slots, which will save us /// gas. ExcludeAndSeal, diff --git a/core/tests/loadnext/README.md b/core/tests/loadnext/README.md index 52b4c68dec34..59288a7160ec 100644 --- a/core/tests/loadnext/README.md +++ b/core/tests/loadnext/README.md @@ -1,7 +1,7 @@ -# Loadnext: loadtest for zkSync +# Loadnext: loadtest for ZKsync -Loadnext is a utility for random stress-testing the zkSync server. It is capable of simulating the behavior of many -independent users of zkSync network, who are sending quasi-random requests to the server. +Loadnext is a utility for random stress-testing the ZKsync server. It is capable of simulating the behavior of many +independent users of ZKsync network, who are sending quasi-random requests to the server. The general flow is as follows: diff --git a/core/tests/loadnext/src/account/mod.rs b/core/tests/loadnext/src/account/mod.rs index d5bd22dd6844..5dcd5167165e 100644 --- a/core/tests/loadnext/src/account/mod.rs +++ b/core/tests/loadnext/src/account/mod.rs @@ -354,7 +354,7 @@ impl AccountLifespan { } } - /// Generic submitter for zkSync network: it can operate individual transactions, + /// Generic submitter for ZKsync network: it can operate individual transactions, /// as long as we can provide a `SyncTransactionHandle` to wait for the commitment and the /// execution result. /// Once result is obtained, it's compared to the expected operation outcome in order to check whether diff --git a/core/tests/loadnext/src/account_pool.rs b/core/tests/loadnext/src/account_pool.rs index 1ea6f61b9dff..7b5e277e139b 100644 --- a/core/tests/loadnext/src/account_pool.rs +++ b/core/tests/loadnext/src/account_pool.rs @@ -77,7 +77,7 @@ pub struct TestWallet { } /// Pool of accounts to be used in the test. -/// Each account is represented as `zksync::Wallet` in order to provide convenient interface of interaction with zkSync. +/// Each account is represented as `zksync::Wallet` in order to provide convenient interface of interaction with ZKsync. #[derive(Debug)] pub struct AccountPool { /// Main wallet that will be used to initialize all the test wallets. @@ -102,7 +102,7 @@ impl AccountPool { )? .for_network(l2_chain_id.into()) .build(); - // Perform a health check: check whether zkSync server is alive. + // Perform a health check: check whether ZKsync server is alive. let mut server_alive = false; for _ in 0usize..3 { if let Ok(Ok(_)) = timeout(Duration::from_secs(3), client.get_main_contract()).await { @@ -111,7 +111,7 @@ impl AccountPool { } } if !server_alive { - anyhow::bail!("zkSync server does not respond. Please check RPC address and whether server is launched"); + anyhow::bail!("ZKsync server does not respond. Please check RPC address and whether server is launched"); } let test_contract = loadnext_contract(&config.test_contracts_path)?; diff --git a/core/tests/loadnext/src/command/api.rs b/core/tests/loadnext/src/command/api.rs index b32620bd3438..2f5628f5759b 100644 --- a/core/tests/loadnext/src/command/api.rs +++ b/core/tests/loadnext/src/command/api.rs @@ -53,7 +53,7 @@ impl AllWeighted for ApiRequestType { pub struct ApiRequest { /// Type of the request to be performed. pub request_type: ApiRequestType, - /// zkSync block number, generated randomly. + /// ZKsync block number, generated randomly. pub block_number: api::BlockNumber, } diff --git a/core/tests/loadnext/src/command/tx_command.rs b/core/tests/loadnext/src/command/tx_command.rs index a2ac37dfc8b8..2c325f1a67e4 100644 --- a/core/tests/loadnext/src/command/tx_command.rs +++ b/core/tests/loadnext/src/command/tx_command.rs @@ -12,7 +12,7 @@ use crate::{ static WEIGHTS: OnceCell<[(TxType, f32); 5]> = OnceCell::new(); -/// Type of transaction. It doesn't copy the zkSync operation list, because +/// Type of transaction. It doesn't copy the ZKsync operation list, because /// it divides some transactions in subcategories (e.g. to new account / to existing account; to self / to other; etc)/ #[derive(Debug, Copy, Clone, PartialEq)] pub enum TxType { diff --git a/core/tests/loadnext/src/config.rs b/core/tests/loadnext/src/config.rs index 7f3e1e258305..a9648edb00ae 100644 --- a/core/tests/loadnext/src/config.rs +++ b/core/tests/loadnext/src/config.rs @@ -41,7 +41,7 @@ pub struct LoadtestConfig { /// Address of the ERC-20 token to be used in test. /// /// Token must satisfy two criteria: - /// - Be supported by zkSync. + /// - Be supported by ZKsync. /// - Have `mint` operation. /// /// Note that we use ERC-20 token since we can't easily mint a lot of ETH on diff --git a/core/tests/loadnext/src/corrupted_tx.rs b/core/tests/loadnext/src/corrupted_tx.rs index cb1c8fcf1b7b..cf4064a4cf8f 100644 --- a/core/tests/loadnext/src/corrupted_tx.rs +++ b/core/tests/loadnext/src/corrupted_tx.rs @@ -6,7 +6,7 @@ use zksync_types::{ use crate::{command::IncorrectnessModifier, sdk::signer::Signer}; -/// Trait that exists solely to extend the signed zkSync transaction interface, providing the ability +/// Trait that exists solely to extend the signed ZKsync transaction interface, providing the ability /// to modify transaction in a way that will make it invalid. /// /// Loadtest is expected to simulate the user behavior, and it's not that uncommon of users to send incorrect diff --git a/core/tests/loadnext/src/executor.rs b/core/tests/loadnext/src/executor.rs index a7b1fa47c994..48d90f19c1d7 100644 --- a/core/tests/loadnext/src/executor.rs +++ b/core/tests/loadnext/src/executor.rs @@ -633,7 +633,7 @@ impl Executor { /// Returns the amount of funds to be deposited on the main account in L2. /// Amount is chosen to be big enough to not worry about precisely calculating the remaining balances on accounts, - /// but also to not be close to the supported limits in zkSync. + /// but also to not be close to the supported limits in ZKsync. fn amount_to_deposit(&self) -> u128 { u128::MAX >> 32 } @@ -696,7 +696,7 @@ async fn deposit_with_attempts( tracing::info!("Deposit with tx_hash {deposit_tx_hash:?}"); - // Wait for the corresponding priority operation to be committed in zkSync. + // Wait for the corresponding priority operation to be committed in ZKsync. match ethereum.wait_for_tx(deposit_tx_hash).await { Ok(eth_receipt) => { return Ok(eth_receipt); diff --git a/core/tests/loadnext/src/main.rs b/core/tests/loadnext/src/main.rs index 6a3125931f12..309dd7557687 100644 --- a/core/tests/loadnext/src/main.rs +++ b/core/tests/loadnext/src/main.rs @@ -1,8 +1,8 @@ -//! Loadtest: an utility to stress-test the zkSync server. +//! Loadtest: an utility to stress-test the ZKsync server. //! //! In order to launch it, you must provide required environmental variables, for details see `README.md`. //! Without required variables provided, test is launched in the localhost/development mode with some hard-coded -//! values to check the local zkSync deployment. +//! values to check the local ZKsync deployment. use std::time::Duration; diff --git a/core/tests/loadnext/src/sdk/abi/update-abi.sh b/core/tests/loadnext/src/sdk/abi/update-abi.sh index 35d03a469dfd..3fdcd4d58028 100755 --- a/core/tests/loadnext/src/sdk/abi/update-abi.sh +++ b/core/tests/loadnext/src/sdk/abi/update-abi.sh @@ -2,7 +2,7 @@ cd `dirname $0` -# Main zkSync contract interface +# Main ZKsync contract interface cat $ZKSYNC_HOME/contracts/l1-contracts/artifacts/contracts/bridgehub/IBridgehub.sol/IBridgehub.json | jq '{ abi: .abi}' > IBridgehub.json cat $ZKSYNC_HOME/contracts/l1-contracts/artifacts/contracts/state-transition/IStateTransitionManager.sol/IStateTransitionManager.json | jq '{ abi: .abi}' > IStateTransitionManager.json cat $ZKSYNC_HOME/contracts/l1-contracts/artifacts/contracts/state-transition/chain-interfaces/IZkSyncHyperchain.sol/IZkSyncHyperchain.json | jq '{ abi: .abi}' > IZkSyncHyperchain.json diff --git a/core/tests/loadnext/src/sdk/ethereum/mod.rs b/core/tests/loadnext/src/sdk/ethereum/mod.rs index 6800fb75a7d3..ca168152a640 100644 --- a/core/tests/loadnext/src/sdk/ethereum/mod.rs +++ b/core/tests/loadnext/src/sdk/ethereum/mod.rs @@ -135,7 +135,7 @@ impl EthereumProvider { self.eth_client.as_ref() } - /// Returns the zkSync contract address. + /// Returns the ZKsync contract address. pub fn contract_address(&self) -> H160 { self.client().contract_addr() } @@ -272,7 +272,7 @@ impl EthereumProvider { } /// Performs a transfer of funds from one Ethereum account to another. - /// Note: This operation is performed on Ethereum, and not related to zkSync directly. + /// Note: This operation is performed on Ethereum, and not related to ZKsync directly. pub async fn transfer( &self, token_address: Address, @@ -443,7 +443,7 @@ impl EthereumProvider { Ok(tx_hash) } - /// Performs a deposit in zkSync network. + /// Performs a deposit in ZKsync network. /// For ERC20 tokens, a deposit must be approved beforehand via the `EthereumProvider::approve_erc20_token_deposits` method. #[allow(clippy::too_many_arguments)] pub async fn deposit( diff --git a/core/tests/ts-integration/README.md b/core/tests/ts-integration/README.md index cb3a1aa5ae7e..93c225066698 100644 --- a/core/tests/ts-integration/README.md +++ b/core/tests/ts-integration/README.md @@ -1,6 +1,6 @@ # NFTF -- New Fancy Test Framework -This folder contains a framework for writing integration tests for zkSync Era, as well as set of integration test +This folder contains a framework for writing integration tests for ZKsync Era, as well as set of integration test suites. This framework is built atop of [jest](https://jestjs.io/). It is _highly recommended_ to familiarize yourself with its @@ -23,7 +23,7 @@ prepare the context for tests. Context initialization consists of: - Creating personal accounts for each test suite. - Providing funds to these accounts. -Basically, during initialization, everything is prepared for writing tests that interact with zkSync. +Basically, during initialization, everything is prepared for writing tests that interact with ZKsync. After that, each test suite is ran _in parallel_. Each test suite can claim its own account and be sure that this account has funds on it and is not used by any other suite. diff --git a/core/tests/ts-integration/contracts/README.md b/core/tests/ts-integration/contracts/README.md index d08f934e8456..532703ad210f 100644 --- a/core/tests/ts-integration/contracts/README.md +++ b/core/tests/ts-integration/contracts/README.md @@ -1,4 +1,4 @@ # Contracts test data This folder contains data for contracts that are being used for testing to check the correctness of the smart contract -flow in zkSync. +flow in ZKsync. diff --git a/core/tests/ts-integration/contracts/custom-account/SystemContractsCaller.sol b/core/tests/ts-integration/contracts/custom-account/SystemContractsCaller.sol index 01b7b5198add..c5be4983e377 100644 --- a/core/tests/ts-integration/contracts/custom-account/SystemContractsCaller.sol +++ b/core/tests/ts-integration/contracts/custom-account/SystemContractsCaller.sol @@ -6,7 +6,7 @@ import {MSG_VALUE_SIMULATOR_IS_SYSTEM_BIT, MSG_VALUE_SYSTEM_CONTRACT} from "./Co import "./Utils.sol"; // Addresses used for the compiler to be replaced with the -// zkSync-specific opcodes during the compilation. +// ZKsync-specific opcodes during the compilation. // IMPORTANT: these are just compile-time constants and are used // only if used in-place by Yul optimizer. address constant TO_L1_CALL_ADDRESS = address((1 << 16) - 1); diff --git a/core/tests/ts-integration/contracts/custom-account/TransactionHelper.sol b/core/tests/ts-integration/contracts/custom-account/TransactionHelper.sol index 7fc883ed882c..82747b88d358 100644 --- a/core/tests/ts-integration/contracts/custom-account/TransactionHelper.sol +++ b/core/tests/ts-integration/contracts/custom-account/TransactionHelper.sol @@ -10,7 +10,7 @@ import "./interfaces/IContractDeployer.sol"; import {BASE_TOKEN_SYSTEM_CONTRACT, BOOTLOADER_FORMAL_ADDRESS} from "./Constants.sol"; import "./RLPEncoder.sol"; -/// @dev The type id of zkSync's EIP-712-signed transaction. +/// @dev The type id of ZKsync's EIP-712-signed transaction. uint8 constant EIP_712_TX_TYPE = 0x71; /// @dev The type id of legacy transactions. @@ -20,7 +20,7 @@ uint8 constant EIP_2930_TX_TYPE = 0x01; /// @dev The type id of EIP1559 transactions. uint8 constant EIP_1559_TX_TYPE = 0x02; -/// @notice Structure used to represent zkSync transaction. +/// @notice Structure used to represent ZKsync transaction. struct Transaction { // The type of the transaction. uint256 txType; @@ -118,7 +118,7 @@ library TransactionHelper { } } - /// @notice Encode hash of the zkSync native transaction type. + /// @notice Encode hash of the ZKsync native transaction type. /// @return keccak256 hash of the EIP-712 encoded representation of transaction function _encodeHashEIP712Transaction(Transaction calldata _transaction) private @@ -251,7 +251,7 @@ library TransactionHelper { // Hash of EIP2930 transactions is encoded the following way: // H(0x01 || RLP(chain_id, nonce, gas_price, gas_limit, destination, amount, data, access_list)) // - // Note, that on zkSync access lists are not supported and should always be empty. + // Note, that on ZKsync access lists are not supported and should always be empty. // Encode all fixed-length params to avoid "stack too deep error" bytes memory encodedFixedLengthParams; @@ -290,7 +290,7 @@ library TransactionHelper { // Otherwise the length is not encoded at all. } - // On zkSync, access lists are always zero length (at least for now). + // On ZKsync, access lists are always zero length (at least for now). bytes memory encodedAccessListLength = RLPEncoder.encodeListLen(0); bytes memory encodedListLength; @@ -327,7 +327,7 @@ library TransactionHelper { // Hash of EIP1559 transactions is encoded the following way: // H(0x02 || RLP(chain_id, nonce, max_priority_fee_per_gas, max_fee_per_gas, gas_limit, destination, amount, data, access_list)) // - // Note, that on zkSync access lists are not supported and should always be empty. + // Note, that on ZKsync access lists are not supported and should always be empty. // Encode all fixed-length params to avoid "stack too deep error" bytes memory encodedFixedLengthParams; @@ -368,7 +368,7 @@ library TransactionHelper { // Otherwise the length is not encoded at all. } - // On zkSync, access lists are always zero length (at least for now). + // On ZKsync, access lists are always zero length (at least for now). bytes memory encodedAccessListLength = RLPEncoder.encodeListLen(0); bytes memory encodedListLength; diff --git a/core/tests/ts-integration/contracts/custom-account/Utils.sol b/core/tests/ts-integration/contracts/custom-account/Utils.sol index da3d4eb60878..e562948942d7 100644 --- a/core/tests/ts-integration/contracts/custom-account/Utils.sol +++ b/core/tests/ts-integration/contracts/custom-account/Utils.sol @@ -3,7 +3,7 @@ pragma solidity >=0.8.0; /** * @author Matter Labs - * @dev Common utilities used in zkSync system contracts + * @dev Common utilities used in ZKsync system contracts */ library Utils { function safeCastToU128(uint256 _x) internal pure returns (uint128) { diff --git a/core/tests/ts-integration/src/helpers.ts b/core/tests/ts-integration/src/helpers.ts index d3464bc84bdd..7848749bfe31 100644 --- a/core/tests/ts-integration/src/helpers.ts +++ b/core/tests/ts-integration/src/helpers.ts @@ -64,7 +64,7 @@ export async function anyTransaction(wallet: zksync.Wallet): Promise { }); test('Should check the network version', async () => { - // Valid network IDs for zkSync are greater than 270. + // Valid network IDs for ZKsync are greater than 270. // This test suite may run on different envs, so we don't expect a particular ID. await expect(alice.provider.send('net_version', [])).resolves.toMatch(chainId.toString()); }); diff --git a/core/tests/ts-integration/tests/contracts.test.ts b/core/tests/ts-integration/tests/contracts.test.ts index 57e9ad057506..2b23ab7cb34b 100644 --- a/core/tests/ts-integration/tests/contracts.test.ts +++ b/core/tests/ts-integration/tests/contracts.test.ts @@ -154,7 +154,7 @@ describe('Smart contract behavior checks', () => { test('Should interchangeably use ethers for eth calls', async () => { // In this test we make sure that we can use `ethers` `Contract` object and provider - // to do an `eth_Call` and send transactions to zkSync contract. + // to do an `eth_Call` and send transactions to ZKsync contract. // This check is important to ensure that external apps do not have to use our SDK and // can keep using `ethers` on their side. diff --git a/core/tests/ts-integration/tests/l1.test.ts b/core/tests/ts-integration/tests/l1.test.ts index db0308ba4b9d..e149a8f7e59f 100644 --- a/core/tests/ts-integration/tests/l1.test.ts +++ b/core/tests/ts-integration/tests/l1.test.ts @@ -149,7 +149,7 @@ describe('Tests for L1 behavior', () => { const accumutatedRoot = calculateAccumulatedRoot(alice.address, message, receipt.l1BatchTxIndex, id, proof); expect(accumutatedRoot).toBe(root); - // Ensure that provided proof is accepted by the main zkSync contract. + // Ensure that provided proof is accepted by the main ZKsync contract. const chainContract = await alice.getMainContract(); const acceptedByContract = await chainContract.proveL2MessageInclusion( receipt.l1BatchNumber, diff --git a/core/tests/ts-integration/tests/mempool.test.ts b/core/tests/ts-integration/tests/mempool.test.ts index 00f95bfefacb..6dacc54ac1fb 100644 --- a/core/tests/ts-integration/tests/mempool.test.ts +++ b/core/tests/ts-integration/tests/mempool.test.ts @@ -137,7 +137,7 @@ describe('Tests for the mempool behavior', () => { }); /** - * Sends a valid zkSync transaction with a certain nonce. + * Sends a valid ZKsync transaction with a certain nonce. * What transaction does is assumed to be not important besides the fact that it should be accepted. * * @param wallet Wallet to send transaction from. diff --git a/core/tests/ts-integration/tests/self-unit.test.ts b/core/tests/ts-integration/tests/self-unit.test.ts index f59d66f1361f..50655e7c2c73 100644 --- a/core/tests/ts-integration/tests/self-unit.test.ts +++ b/core/tests/ts-integration/tests/self-unit.test.ts @@ -1,6 +1,6 @@ /** * This file contains unit tests for the framework itself. - * It does not receive a funced account and should not interact with the zkSync server. + * It does not receive a funced account and should not interact with the ZKsync server. */ import { TestMaster } from '../src/index'; import { BigNumber } from 'ethers'; diff --git a/docs/guides/advanced/01_initialization.md b/docs/guides/advanced/01_initialization.md index 7e7e74957cb8..79c33434d3b5 100644 --- a/docs/guides/advanced/01_initialization.md +++ b/docs/guides/advanced/01_initialization.md @@ -1,6 +1,6 @@ -# zkSync deeper dive +# ZKsync deeper dive -The goal of this doc is to show you some more details on how zkSync works internally. +The goal of this doc is to show you some more details on how ZKsync works internally. Please do the dev_setup.md and development.md (these commands do all the heavy lifting on starting the components of the system). @@ -20,9 +20,9 @@ there, make sure to run `zk` (that compiles this code), before re-running `zk in As first step, it gets the docker images for postgres and reth. -Reth (one of the Ethereum clients) will be used to setup our own copy of L1 chain (that our local zkSync would use). +Reth (one of the Ethereum clients) will be used to setup our own copy of L1 chain (that our local ZKsync would use). -Postgres is one of the two databases, that is used by zkSync (the other one is RocksDB). Currently most of the data is +Postgres is one of the two databases, that is used by ZKsync (the other one is RocksDB). Currently most of the data is stored in postgres (blocks, transactions etc) - while RocksDB is only storing the state (Tree & Map) - and it used by VM. @@ -116,7 +116,7 @@ This is one of the "rich wallets" we predefined for local L1. **Note:** This reth shell is running official Ethereum JSON RPC with Reth-specific extensions documented at [reth docs](https://paradigmxyz.github.io/reth/jsonrpc/intro.html) -In order to communicate with L2 (our zkSync) - we have to deploy multiple contracts onto L1 (our local reth created +In order to communicate with L2 (our ZKsync) - we have to deploy multiple contracts onto L1 (our local reth created Ethereum). You can look on the `deployL1.log` file - to see the list of contracts that were deployed and their accounts. First thing in the file, is the deployer/governor wallet - this is the account that can change, freeze and unfreeze the diff --git a/docs/guides/advanced/02_deposits.md b/docs/guides/advanced/02_deposits.md index 7a40e33f91c9..4018fed46325 100644 --- a/docs/guides/advanced/02_deposits.md +++ b/docs/guides/advanced/02_deposits.md @@ -33,7 +33,7 @@ Now, let's see how many tokens we have: // This checks the tokens on 'L1' (reth) ./web3 --rpc-url http://localhost:8545 balance 0x618263CE921F7dd5F4f40C29f6c524Aaf97b9bbd -// This checks the tokens on 'L2' (zkSync) +// This checks the tokens on 'L2' (ZKsync) ./web3 --rpc-url http://localhost:3050 balance 0x618263CE921F7dd5F4f40C29f6c524Aaf97b9bbd ``` @@ -55,7 +55,7 @@ and now let's bridge it over to L2. ## Bridging over to L2 -For an easy way to bridge we'll use [zkSync CLI](https://github.com/matter-labs/zksync-cli) +For an easy way to bridge we'll use [ZKsync CLI](https://github.com/matter-labs/zksync-cli) ```shell npx zksync-cli bridge deposit --chain=dockerized-node --amount 3 --pk=0x5090c024edb3bdf4ce2ebc2da96bedee925d9d77d729687e5e2d56382cf0a5a6 --to=0x618263CE921F7dd5F4f40C29f6c524Aaf97b9bbd diff --git a/docs/guides/advanced/03_withdrawals.md b/docs/guides/advanced/03_withdrawals.md index 3d1a46ff4cb9..69f5b0f87089 100644 --- a/docs/guides/advanced/03_withdrawals.md +++ b/docs/guides/advanced/03_withdrawals.md @@ -1,4 +1,4 @@ -# zkSync deeper dive bridging stuff back (a.k.a withdrawals) +# ZKsync deeper dive bridging stuff back (a.k.a withdrawals) Assuming that you have completed [part 1](01_initialization.md) and [part 2](02_deposits.md) already, we can bridge the tokens back by simply calling the zksync-cli: diff --git a/docs/guides/advanced/0_alternative_vm_intro.md b/docs/guides/advanced/0_alternative_vm_intro.md index b47c71bde2f5..fab623e38ae3 100644 --- a/docs/guides/advanced/0_alternative_vm_intro.md +++ b/docs/guides/advanced/0_alternative_vm_intro.md @@ -4,7 +4,7 @@ [Back to ToC](../../specs/README.md) -The zkSync zkEVM plays a fundamentally different role in the zkStack than the EVM does in Ethereum. The EVM is used to +The ZKsync zkEVM plays a fundamentally different role in the zkStack than the EVM does in Ethereum. The EVM is used to execute code in Ethereum's state transition function. This STF needs a client to implement and run it. Ethereum has a multi-client philosophy, there are multiple clients, and they are written in Go, Rust, and other traditional programming languages, all running and verifying the same STF. @@ -68,7 +68,7 @@ For each frame, the following memory areas are allocated: calldata/copy the `returndata` from the calls to system contracts to not interfere with the standard Solidity memory alignment. - _Stack_. Unlike Ethereum, stack is not the primary place to get arguments for opcodes. The biggest difference between - stack on zkEVM and EVM is that on zkSync stack can be accessed at any location (just like memory). While users do not + stack on zkEVM and EVM is that on ZKsync stack can be accessed at any location (just like memory). While users do not pay for the growth of stack, the stack can be fully cleared at the end of the frame, so the overhead is minimal. - _Code_. The memory area from which the VM executes the code of the contract. The contract itself can not read the code page, it is only done implicitly by the VM. @@ -115,7 +115,7 @@ copying. Some of the operations which are opcodes on Ethereum, have become calls to some of the system contracts. The most notable examples are `Keccak256`, `SystemContext`, etc. Note, that, if done naively, the following lines of code would -work differently on zkSync and Ethereum: +work differently on ZKsync and Ethereum: ```solidity pop(call(...)) @@ -142,7 +142,7 @@ result in `revert(0,0)`. - `mimic_call`. The same as a normal `call`, but it can alter the `msg.sender` field of the transaction. - `to_l1`. Sends a system L2→L1 log to Ethereum. The structure of this log can be seen [here](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/contracts/ethereum/contracts/zksync/Storage.sol#L47). -- `event`. Emits an L2 log to zkSync. Note, that L2 logs are not equivalent to Ethereum events. Each L2 log can emit 64 +- `event`. Emits an L2 log to ZKsync. Note, that L2 logs are not equivalent to Ethereum events. Each L2 log can emit 64 bytes of data (the actual size is 88 bytes, because it includes the emitter address, etc). A single Ethereum event is represented with multiple `event` logs constitute. This opcode is only used by `EventWriter` system contract. - `precompile_call`. This is an opcode that accepts two parameters: the uint256 representing the packed parameters for @@ -227,7 +227,7 @@ by another system contract (since Matter Labs is fully aware of system contracts ### Simulations via our compiler In the future, we plan to introduce our “extended” version of Solidity with more supported opcodes than the original -one. However, right now it was beyond the capacity of the team to do, so in order to represent accessing zkSync-specific +one. However, right now it was beyond the capacity of the team to do, so in order to represent accessing ZKsync-specific opcodes, we use `call` opcode with certain constant parameters that will be automatically replaced by the compiler with zkEVM native opcode. @@ -251,7 +251,7 @@ Full list of opcode simulations can be found We also use [verbatim-like](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/verbatim.md) -statements to access zkSync-specific opcodes in the bootloader. +statements to access ZKsync-specific opcodes in the bootloader. All the usages of the simulations in our Solidity code are implemented in the [SystemContractHelper](https://github.com/code-423n4/2023-10-zksync/blob/main/code/system-contracts/contracts/libraries/SystemContractHelper.sol) @@ -284,7 +284,7 @@ above substitutions to work. ## Bytecode hashes -On zkSync the bytecode hashes are stored in the following format: +On ZKsync the bytecode hashes are stored in the following format: - The 0th byte denotes the version of the format. Currently the only version that is used is “1”. - The 1st byte is `0` for deployed contracts’ code and `1` for the contract code @@ -306,6 +306,6 @@ Note, that it does not have to consist of only correct opcodes. In case the VM e simply revert (similar to how EVM would treat them). A call to a contract with invalid bytecode can not be proven. That is why it is **essential** that no contract with -invalid bytecode is ever deployed on zkSync. It is the job of the +invalid bytecode is ever deployed on ZKsync. It is the job of the [KnownCodesStorage](https://github.com/matter-labs/zksync-era/blob/main/docs/specs/zk_evm/system_contracts.md#knowncodestorage) to ensure that all allowed bytecodes in the system are valid. diff --git a/docs/guides/advanced/contracts.md b/docs/guides/advanced/contracts.md index 03d09469975d..5148ee917f7f 100644 --- a/docs/guides/advanced/contracts.md +++ b/docs/guides/advanced/contracts.md @@ -1,10 +1,10 @@ -# zkSync contracts +# ZKsync contracts -Now that we know how to bridge tokens back and forth, let's talk about running things on zkSync. +Now that we know how to bridge tokens back and forth, let's talk about running things on ZKsync. We have a bunch of great tutorials (like this one ) that you can follow to get the exact code & command line calls to create the contracts - so in this article, let's focus on -how things differ between zkSync and Ethereum. +how things differ between ZKsync and Ethereum. **Note** Before reading this article, I'd recommend doing the hardhat tutorial above. @@ -23,9 +23,9 @@ the ABI, so that they can set the proper function arguments). All the bytecode will be run on the EVM (Ethereum Virtual Machine) - that has a stack, access to memory and storage, and a bunch of opcodes. -## zkSync flow +## ZKsync flow -The main part (and the main cost) of the zkSync is the proving system. In order to make proof as fast as possible, we're +The main part (and the main cost) of the ZKsync is the proving system. In order to make proof as fast as possible, we're running a little bit different virtual machine (zkEVM) - that has a slightly different set of opcodes, and also contains a bunch of registers. More details on this will be written in the future articles. @@ -37,7 +37,7 @@ While having a separate compiler introduces a bunch of challenges (for example, allows us to move some of the VM logic (like new contract deployment) into System contracts - which allows faster & cheaper modifications and increased flexibility. -### zkSync system contracts +### ZKsync system contracts Small note on system contracts: as mentioned above, we moved some of the VM logic into system contracts, which allows us to keep VM simpler (and with this - keep the proving system simpler). @@ -51,15 +51,15 @@ visible - like our `ContractDeployer` ### ContractDeployer -Deploying a new contract differs on Ethereum and zkSync. +Deploying a new contract differs on Ethereum and ZKsync. -While on Ethereum - you send the transaction to 0x00 address - on zkSync you have to call the special `ContractDeployer` +While on Ethereum - you send the transaction to 0x00 address - on ZKsync you have to call the special `ContractDeployer` system contract. If you look on your hardhat example, you'll notice that your `deploy.ts` is actually using a `Deployer` class from the `hardhat-zksync-deploy` plugin. -Which inside uses the zkSync's web3.js, that calls the contract deployer +Which inside uses the ZKsync's web3.js, that calls the contract deployer [here](https://github.com/zksync-sdk/zksync2-js/blob/b1d11aa016d93ebba240cdeceb40e675fb948133/src/contract.ts#L76) ```typescript @@ -71,14 +71,14 @@ override getDeployTransaction(..) { ``` Also `ContractDeployer` adding a special prefix for all the new contract addresses. This means that contract addresses -WILL be different on `zkSync` and Ethereum (and also leaves us the possibility of adding Ethereum addresses in the +WILL be different on `ZKsync` and Ethereum (and also leaves us the possibility of adding Ethereum addresses in the future if needed). You can look for `CREATE2_PREFIX` and `CREATE_PREFIX` in the code. ### Gas costs -Another part, where zkSync differs from Ethereum is gas cost. The best example for this are storage slots. +Another part, where ZKsync differs from Ethereum is gas cost. The best example for this are storage slots. If you have two transactions that are updating the same storage slot - and they are in the same 'batch' - only the first one would be charged (as when we write the final storage to ethereum, we just write the final diff of what slots have @@ -86,11 +86,11 @@ changed - so updating the same slot multiple times doesn't increase the amount o ### Account abstraction and some method calls -As `zkSync` has a built-in Account Abstraction (more on this in a separate article) - you shouldn't depend on some of +As `ZKsync` has a built-in Account Abstraction (more on this in a separate article) - you shouldn't depend on some of the solidity functions (like `ecrecover` - that checks the keys, or `tx.origin`) - in all the cases, the compiler will try to warn you. ## Summary -In this article, we looked at how contract development & deployment differs on Ethereum and zkSync (looking at +In this article, we looked at how contract development & deployment differs on Ethereum and ZKsync (looking at differences in VMs, compilers and system contracts). diff --git a/docs/guides/advanced/fee_model.md b/docs/guides/advanced/fee_model.md index 40974f461ea3..3e6473d3ab96 100644 --- a/docs/guides/advanced/fee_model.md +++ b/docs/guides/advanced/fee_model.md @@ -182,7 +182,7 @@ meaning it cannot be less than 80M or more than 4G. ### Why validation is special In Ethereum, there is a fixed cost for verifying a transaction's correctness by checking its signature. However, in -zkSync, due to Account Abstraction, we may need to execute some contract code to determine whether it's ready to accept +ZKsync, due to Account Abstraction, we may need to execute some contract code to determine whether it's ready to accept the transaction. If the contract rejects the transaction, it must be dropped, and there's no one to charge for that process. @@ -224,7 +224,7 @@ You can find this code in [get_txs_fee_in_wei][get_txs_fee_in_wei] function. ## Q&A -### Is zkSync really cheaper +### Is ZKsync really cheaper In short, yes. As seen in the table at the beginning, the regular L2 gas price is set to 0.25 Gwei, while the standard Ethereum price is around 60-100 Gwei. However, the cost of publishing to L1 depends on L1 prices, meaning that the @@ -232,7 +232,7 @@ actual transaction costs will increase if the L1 gas price rises. ### Why do I hear about large refunds -There are a few reasons why refunds might be 'larger' on zkSync (i.e., why we might be overestimating the fees): +There are a few reasons why refunds might be 'larger' on ZKsync (i.e., why we might be overestimating the fees): - We must assume (pessimistically) that you'll have to pay for all the slot/storage writes. In practice, if multiple transactions touch the same slot, we only charge one of them. diff --git a/docs/guides/advanced/how_l2_messaging_works.md b/docs/guides/advanced/how_l2_messaging_works.md index 7bd067eca550..45aba51da03b 100644 --- a/docs/guides/advanced/how_l2_messaging_works.md +++ b/docs/guides/advanced/how_l2_messaging_works.md @@ -1,6 +1,6 @@ # How L2 to L1 messaging works -In this article, we will explore the workings of Layer 2 (L2) to Layer 1 (L1) messaging in zkSync Era. +In this article, we will explore the workings of Layer 2 (L2) to Layer 1 (L1) messaging in ZKsync Era. If you're uncertain about why messaging is necessary in the first place, please refer to our [user documentation][user_docs]. diff --git a/docs/guides/advanced/how_transaction_works.md b/docs/guides/advanced/how_transaction_works.md index 800b2612d162..96c75e3609cc 100644 --- a/docs/guides/advanced/how_transaction_works.md +++ b/docs/guides/advanced/how_transaction_works.md @@ -26,7 +26,7 @@ Here's a simplified table of the transaction types: | 0x0 | 'Legacy' | Only includes `gas price` | These are traditional Ethereum transactions. | 60% / 82% | | 0x1 | EIP-2930 | Contains a list of storage keys/addresses the transaction will access | At present, this type of transaction is not enabled. | | 0x2 | EIP-1559 | Includes `max_priority_fee_per_gas`, `max_gas_price` | These are Ethereum transactions that provide more control over the gas fee. | 35% / 12% | -| 0x71 | EIP-712 (specific to zkSync) | Similar to EIP-1559, but also adds `max_gas_per_pubdata`, custom signatures, and Paymaster support | This is used by those who are using zkSync specific Software Development Kits (SDKs). | 1% / 2% | +| 0x71 | EIP-712 (specific to ZKsync) | Similar to EIP-1559, but also adds `max_gas_per_pubdata`, custom signatures, and Paymaster support | This is used by those who are using ZKsync specific Software Development Kits (SDKs). | 1% / 2% | | 0xFF | L1 transactions also known as priority transactions `L1Tx` | Originating from L1, these have more custom fields like 'refund' addresses etc | Mainly used to transfer funds/data between L1 & L2 layer. | 4% / 3% | Here's the code that does the parsing: [TransactionRequest::from_bytes][transaction_request_from_bytes] diff --git a/docs/guides/advanced/pubdata-with-blobs.md b/docs/guides/advanced/pubdata-with-blobs.md index e27372e934ef..edeaa5b4ebf0 100644 --- a/docs/guides/advanced/pubdata-with-blobs.md +++ b/docs/guides/advanced/pubdata-with-blobs.md @@ -16,11 +16,11 @@ unlike 4844 which supports just 6 per block. ## Technical Approach -The approach spans both L2 system contracts and L1 zkSync contracts (namely `Executor.sol`). When a batch is sealed on +The approach spans both L2 system contracts and L1 ZKsync contracts (namely `Executor.sol`). When a batch is sealed on L2 we will chunk it into blob-sized pieces (4096 elements \* 31 bytes per what is required by our circuits), take the hash of each chunk, and send them to L1 via system logs. Within `Executor.sol` , when we are dealing with blob-based commitments, we verify that the blob contains the correct data with the point evaluation precompile. If the batch -utilizes calldata instead, the processing should remain the same as in a pre-4844 zkSync. Regardless of if pubdata is in +utilizes calldata instead, the processing should remain the same as in a pre-4844 ZKsync. Regardless of if pubdata is in calldata or blobs are used, the batch’s commitment changes as we include new data within the auxiliary output. Given that this is the first step to a longer-term solution, and the restrictions of proto-danksharding that get lifted diff --git a/docs/guides/advanced/pubdata.md b/docs/guides/advanced/pubdata.md index cc0c82497cab..7a32076221f1 100644 --- a/docs/guides/advanced/pubdata.md +++ b/docs/guides/advanced/pubdata.md @@ -1,6 +1,6 @@ # Overview -Pubdata in zkSync can be divided up into 4 different categories: +Pubdata in ZKsync can be divided up into 4 different categories: 1. L2 to L1 Logs 2. L2 to L1 Messages @@ -15,7 +15,7 @@ array. > Note: When the 4844 was integrated this bytes array was moved from being part of the calldata to blob data. While the structure of the pubdata changes, we can use the same strategy to pull the relevant information. First, we -need to filter all of the transactions to the L1 zkSync contract for only the `commitBlocks/commitBatches` transactions +need to filter all of the transactions to the L1 ZKsync contract for only the `commitBlocks/commitBatches` transactions where the proposed block has been referenced by a corresponding `executeBlocks/executeBatches` call (the reason for this is that a committed or even proven block can be reverted but an executed one cannot). Once we have all the committed blocks that have been executed, we then will pull the transaction input and the relevant fields, applying them in order diff --git a/docs/guides/architecture.md b/docs/guides/architecture.md index e87f4bca7e55..25676ad74aa7 100644 --- a/docs/guides/architecture.md +++ b/docs/guides/architecture.md @@ -1,7 +1,7 @@ -# zkSync v2 Project Architecture +# ZKsync v2 Project Architecture This document will help you answer the question: _where can I find the logic for x?_ by giving a directory-tree style -structure of the physical architecture of the zkSync Era project. +structure of the physical architecture of the ZKsync Era project. ## High-Level Overview @@ -10,15 +10,15 @@ The zksync-2-dev repository has the following main units: **Smart Contracts:** All the smart contracts in charge of the protocols on the L1 & L2. Some main contracts: - L1 & L2 bridge contracts. -- The zkSync rollup contract on Ethereum. +- The ZKsync rollup contract on Ethereum. - The L1 proof verifier contract. -**Core App:** The execution layer. A node running the zkSync network in charge of the following components: +**Core App:** The execution layer. A node running the ZKsync network in charge of the following components: - Monitoring the L1 smart contract for deposits or priority operations. - Maintaining a mempool that receives transactions. - Picking up transactions from the mempool, executing them in a VM, and changing the state accordingly. -- Generating zkSync chain blocks. +- Generating ZKsync chain blocks. - Preparing circuits for executed blocks to be proved. - Submitting blocks and proofs to the L1 smart contract. - Exposing the Ethereum-compatible web3 API. @@ -36,27 +36,27 @@ This section provides a physical map of folders & files in this repository. - `/contracts` - `/ethereum`: Smart contracts deployed on the Ethereum L1. - - `/zksync`: Smart contracts deployed on the zkSync L2. + - `/zksync`: Smart contracts deployed on the ZKsync L2. - `/core` - - `/bin`: Executables for the microservices components comprising zkSync Core Node. + - `/bin`: Executables for the microservices components comprising ZKsync Core Node. - `/admin-tools`: CLI tools for admin operations (e.g. restarting prover jobs). - `/external_node`: A read replica that can sync from the main node. - `/lib`: All the library crates used as dependencies of the binary crates above. - - `/basic_types`: Crate with essential zkSync primitive types. - - `/config`: All the configured values used by the different zkSync apps. + - `/basic_types`: Crate with essential ZKsync primitive types. + - `/config`: All the configured values used by the different ZKsync apps. - `/contracts`: Contains definitions of commonly used smart contracts. - - `/crypto`: Cryptographical primitives used by the different zkSync crates. + - `/crypto`: Cryptographical primitives used by the different ZKsync crates. - `/dal`: Data availability layer - `/migrations`: All the db migrations applied to create the storage layer. - `/src`: Functionality to interact with the different db tables. - `/eth_client`: Module providing an interface to interact with an Ethereum node. - `/eth_signer`: Module to sign messages and txs. - - `/mempool`: Implementation of the zkSync transaction pool. + - `/mempool`: Implementation of the ZKsync transaction pool. - `/merkle_tree`: Implementation of a sparse Merkle tree. - `/mini_merkle_tree`: In-memory implementation of a sparse Merkle tree. - `/multivm`: A wrapper over several versions of VM that have been used by the main node. @@ -65,47 +65,47 @@ This section provides a physical map of folders & files in this repository. - `/queued_job_processor`: An abstraction for async job processing - `/state`: A state keeper responsible for handling transaction execution and creating miniblocks and L1 batches. - `/storage`: An encapsulated database interface. - - `/test_account`: A representation of zkSync account. - - `/types`: zkSync network operations, transactions, and common types. - - `/utils`: Miscellaneous helpers for zkSync crates. - - `/vlog`: zkSync logging utility. + - `/test_account`: A representation of ZKsync account. + - `/types`: ZKsync network operations, transactions, and common types. + - `/utils`: Miscellaneous helpers for ZKsync crates. + - `/vlog`: ZKsync logging utility. - `/vm`: ULightweight out-of-circuit VM interface. - `/web3_decl`: Declaration of the Web3 API. - `zksync_core/src` - `/api_server` Externally facing APIs. - - `/web3`: zkSync implementation of the Web3 API. + - `/web3`: ZKsync implementation of the Web3 API. - `/tx_sender`: Helper module encapsulating the transaction processing logic. - - `/bin`: The executable main starting point for the zkSync server. - - `/consistency_checker`: zkSync watchdog. - - `/eth_sender`: Submits transactions to the zkSync smart contract. + - `/bin`: The executable main starting point for the ZKsync server. + - `/consistency_checker`: ZKsync watchdog. + - `/eth_sender`: Submits transactions to the ZKsync smart contract. - `/eth_watch`: Fetches data from the L1. for L2 censorship resistance. - `/fee_monitor`: Monitors the ratio of fees collected by executing txs over the costs of interacting with Ethereum. - `/fee_ticker`: Module to define the price components of L2 transactions. - `/gas_adjuster`: Module to determine the fees to pay in txs containing blocks submitted to the L1. - `/gas_tracker`: Module for predicting L1 gas cost for the Commit/PublishProof/Execute operations. - - `/metadata_calculator`: Module to maintain the zkSync state tree. + - `/metadata_calculator`: Module to maintain the ZKsync state tree. - `/state_keeper`: The sequencer. In charge of collecting the pending txs from the mempool, executing them in the VM, and sealing them in blocks. - `/witness_generator`: Takes the sealed blocks and generates a _Witness_, the input for the prover containing the circuits to be proved. - - `/tests`: Testing infrastructure for zkSync network. + - `/tests`: Testing infrastructure for ZKsync network. - `/cross_external_nodes_checker`: A tool for checking external nodes consistency against the main node. - - `/loadnext`: An app for load testing the zkSync server. + - `/loadnext`: An app for load testing the ZKsync server. - `/ts-integration`: Integration tests set implemented in TypeScript. -- `/prover`: zkSync prover orchestrator application. +- `/prover`: ZKsync prover orchestrator application. - `/docker`: Project docker files. -- `/bin` & `/infrastructure`: Infrastructure scripts that help to work with zkSync applications. +- `/bin` & `/infrastructure`: Infrastructure scripts that help to work with ZKsync applications. - `/etc`: Configuration files. - - `/env`:`.env` files that contain environment variables for different configurations of zkSync Server / Prover. + - `/env`:`.env` files that contain environment variables for different configurations of ZKsync Server / Prover. - `/keys`: Verification keys for `circuit` module. -- `/sdk`: Implementation of client libraries for the zkSync network in different programming languages. - - `/zksync-rs`: Rust client library for zkSync. +- `/sdk`: Implementation of client libraries for the ZKsync network in different programming languages. + - `/zksync-rs`: Rust client library for ZKsync. diff --git a/docs/guides/development.md b/docs/guides/development.md index 16d497f876ed..5e53877993dd 100644 --- a/docs/guides/development.md +++ b/docs/guides/development.md @@ -1,6 +1,6 @@ # Development guide -This document covers development-related actions in zkSync. +This document covers development-related actions in ZKsync. ## Initializing the project diff --git a/docs/guides/external-node/00_quick_start.md b/docs/guides/external-node/00_quick_start.md index e244268e7845..826c296fcd93 100644 --- a/docs/guides/external-node/00_quick_start.md +++ b/docs/guides/external-node/00_quick_start.md @@ -4,7 +4,7 @@ Install `docker compose` and `Docker` -## Running zkSync node locally +## Running ZKsync node locally To start a mainnet instance, run: @@ -37,7 +37,7 @@ docker compose --file testnet-external-node-docker-compose.yml down --volumes You can see the status of the node (after recovery) in [local grafana dashboard](http://localhost:3000/d/0/external-node). -Those commands start zkSync node locally inside docker. +Those commands start ZKsync node locally inside docker. The HTTP JSON-RPC API can be accessed on port `3060` and WebSocket API can be accessed on port `3061`. @@ -57,7 +57,7 @@ The HTTP JSON-RPC API can be accessed on port `3060` and WebSocket API can be ac > This configuration is only for nodes that use snapshots recovery (the default for docker-compose setup), for > requirements for nodes running from DB dump see > [03_running.md](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/03_running.md). DB dumps -> are a way to start zkSync node with full historical transactions history +> are a way to start ZKsync node with full historical transactions history > [!NOTE] > diff --git a/docs/guides/external-node/01_intro.md b/docs/guides/external-node/01_intro.md index 440d561bc6fe..c9d01d9a87f9 100644 --- a/docs/guides/external-node/01_intro.md +++ b/docs/guides/external-node/01_intro.md @@ -1,17 +1,17 @@ # ZkSync Node Documentation -This documentation explains the basics of the zkSync Node. +This documentation explains the basics of the ZKsync Node. ## Disclaimers -- The zkSync node is in the alpha phase, and should be used with caution. -- The zkSync node is a read-only replica of the main node. We are currently working on decentralizing our infrastructure - by creating a consensus node. The zkSync node is not going to be the consensus node. +- The ZKsync node is in the alpha phase, and should be used with caution. +- The ZKsync node is a read-only replica of the main node. We are currently working on decentralizing our infrastructure + by creating a consensus node. The ZKsync node is not going to be the consensus node. -## What is the zkSync node +## What is the ZKsync node -The zkSync node is a read-replica of the main (centralized) node that can be run by external parties. It functions by -fetching data from the zkSync API and re-applying transactions locally, starting from the genesis block. The zkSync node +The ZKsync node is a read-replica of the main (centralized) node that can be run by external parties. It functions by +fetching data from the ZKsync API and re-applying transactions locally, starting from the genesis block. The ZKsync node shares most of its codebase with the main node. Consequently, when it re-applies transactions, it does so exactly as the main node did in the past. @@ -23,18 +23,18 @@ main node did in the past. ## High-level overview -At a high level, the zkSync node can be seen as an application that has the following modules: +At a high level, the ZKsync node can be seen as an application that has the following modules: - API server that provides the publicly available Web3 interface. - Synchronization layer that interacts with the main node and retrieves transactions and blocks to re-execute. - Sequencer component that actually executes and persists transactions received from the synchronization layer. -- Several checker modules that ensure the consistency of the zkSync node state. +- Several checker modules that ensure the consistency of the ZKsync node state. With the EN, you are able to: -- Locally recreate and verify the zkSync Era mainnet/testnet state. +- Locally recreate and verify the ZKsync Era mainnet/testnet state. - Interact with the recreated state in a trustless way (in a sense that the validity is locally verified, and you should - not rely on a third-party API zkSync Era provides). + not rely on a third-party API ZKsync Era provides). - Use the Web3 API without having to query the main node. - Send L2 transactions (that will be proxied to the main node). @@ -48,7 +48,7 @@ A more detailed overview of the EN's components is provided in the [components]( ## API overview -API exposed by the zkSync node strives to be Web3-compliant. If some method is exposed but behaves differently compared +API exposed by the ZKsync node strives to be Web3-compliant. If some method is exposed but behaves differently compared to Ethereum, it should be considered a bug. Please [report][contact_us] such cases. [contact_us]: https://zksync.io/contact @@ -87,7 +87,7 @@ Available methods: | `eth_getTransactionReceipt` | | | `eth_protocolVersion` | | | `eth_sendRawTransaction` | | -| `eth_syncing` | zkSync node is considered synced if it's less than 11 blocks behind the main node. | +| `eth_syncing` | ZKsync node is considered synced if it's less than 11 blocks behind the main node. | | `eth_coinbase` | Always returns a zero address | | `eth_accounts` | Always returns an empty list | | `eth_getCompilers` | Always returns an empty list | @@ -154,5 +154,5 @@ Always refer to the documentation linked above to see the list of stabilized met ### `en` namespace -This namespace contains methods that zkSync nodes call on the main node while syncing. If this namespace is enabled, +This namespace contains methods that ZKsync nodes call on the main node while syncing. If this namespace is enabled, other ENs can sync from this node. diff --git a/docs/guides/external-node/02_configuration.md b/docs/guides/external-node/02_configuration.md index 336d01479089..5b8b7512eb3e 100644 --- a/docs/guides/external-node/02_configuration.md +++ b/docs/guides/external-node/02_configuration.md @@ -1,7 +1,7 @@ # ZkSync Node Configuration -This document outlines various configuration options for the EN. Currently, the zkSync node requires the definition of -numerous environment variables. To streamline this process, we provide prepared configs for the zkSync Era - for both +This document outlines various configuration options for the EN. Currently, the ZKsync node requires the definition of +numerous environment variables. To streamline this process, we provide prepared configs for the ZKsync Era - for both [mainnet](prepared_configs/mainnet-config.env) and [testnet](prepared_configs/testnet-sepolia-config.env). You can use these files as a starting point and modify only the necessary sections. @@ -10,7 +10,7 @@ default settings.** ## Database -The zkSync node uses two databases: PostgreSQL and RocksDB. +The ZKsync node uses two databases: PostgreSQL and RocksDB. PostgreSQL serves as the main source of truth in the EN, so all the API requests fetch the state from there. The PostgreSQL connection is configured by the `DATABASE_URL`. Additionally, the `DATABASE_POOL_SIZE` variable defines the @@ -22,11 +22,11 @@ recommended to use an NVME SSD for RocksDB. RocksDB requires two variables to be ## L1 Web3 client -zkSync node requires a connection to an Ethereum node. The corresponding env variable is `EN_ETH_CLIENT_URL`. Make sure +ZKsync node requires a connection to an Ethereum node. The corresponding env variable is `EN_ETH_CLIENT_URL`. Make sure to set the URL corresponding to the correct L1 network (L1 mainnet for L2 mainnet and L1 sepolia for L2 testnet). -Note: Currently, the zkSync node makes 2 requests to the L1 per L1 batch, so the Web3 client usage for a synced node -should not be high. However, during the synchronization phase the new batches would be persisted on the zkSync node +Note: Currently, the ZKsync node makes 2 requests to the L1 per L1 batch, so the Web3 client usage for a synced node +should not be high. However, during the synchronization phase the new batches would be persisted on the ZKsync node quickly, so make sure that the L1 client won't exceed any limits (e.g. in case you use Infura). ## Exposed ports @@ -50,12 +50,12 @@ the metrics, leave this port not configured, and the metrics won't be collected. There are variables that allow you to fine-tune the limits of the RPC servers, such as limits on the number of returned entries or the limit for the accepted transaction size. Provided files contain sane defaults that are recommended for -use, but these can be edited, e.g. to make the zkSync node more/less restrictive. +use, but these can be edited, e.g. to make the ZKsync node more/less restrictive. ## JSON-RPC API namespaces There are 7 total supported API namespaces: `eth`, `net`, `web3`, `debug` - standard ones; `zks` - rollup-specific one; -`pubsub` - a.k.a. `eth_subscribe`; `en` - used by zkSync nodes while syncing. You can configure what namespaces you want +`pubsub` - a.k.a. `eth_subscribe`; `en` - used by ZKsync nodes while syncing. You can configure what namespaces you want to enable using `EN_API_NAMESPACES` and specifying namespace names in a comma-separated list. By default, all but the `debug` namespace are enabled. @@ -64,7 +64,7 @@ to enable using `EN_API_NAMESPACES` and specifying namespace names in a comma-se `MISC_LOG_FORMAT` defines the format in which logs are shown: `plain` corresponds to the human-readable format, while the other option is `json` (recommended for deployments). -`RUST_LOG` variable allows you to set up the logs granularity (e.g. make the zkSync node emit fewer logs). You can read +`RUST_LOG` variable allows you to set up the logs granularity (e.g. make the ZKsync node emit fewer logs). You can read about the format [here](https://docs.rs/env_logger/0.10.0/env_logger/#enabling-logging). `MISC_SENTRY_URL` and `MISC_OTLP_URL` variables can be configured to set up Sentry and OpenTelemetry exporters. diff --git a/docs/guides/external-node/03_running.md b/docs/guides/external-node/03_running.md index f6f76271c0c7..5789c34cdaa2 100644 --- a/docs/guides/external-node/03_running.md +++ b/docs/guides/external-node/03_running.md @@ -14,9 +14,9 @@ This configuration is approximate and should be considered as **minimal** requir - 32-core CPU - 64GB RAM - SSD storage (NVME recommended): - - Sepolia Testnet - 10GB zkSync node + 50GB PostgreSQL (at the time of writing, will grow over time, so should be + - Sepolia Testnet - 10GB ZKsync node + 50GB PostgreSQL (at the time of writing, will grow over time, so should be constantly monitored) - - Mainnet - 3TB zkSync node + 8TB PostgreSQL (at the time of writing, will grow over time, so should be constantly + - Mainnet - 3TB ZKsync node + 8TB PostgreSQL (at the time of writing, will grow over time, so should be constantly monitored) - 100 Mbps connection (1 Gbps+ recommended) @@ -36,22 +36,22 @@ it in Docker. There are many of guides on that, [here's one example](https://www.docker.com/blog/how-to-use-the-postgres-docker-official-image/). Note however that if you run PostgresSQL as a stand-alone Docker image (e.g. not in Docker-compose with a network shared -between zkSync node and Postgres), zkSync node won't be able to access Postgres via `localhost` or `127.0.0.1` URLs. To +between ZKsync node and Postgres), ZKsync node won't be able to access Postgres via `localhost` or `127.0.0.1` URLs. To make it work, you'll have to either run it with a `--network host` (on Linux) or use `host.docker.internal` instead of -`localhost` in the zkSync node configuration ([official docs][host_docker_internal]). +`localhost` in the ZKsync node configuration ([official docs][host_docker_internal]). Besides running Postgres, you are expected to have a DB dump from a corresponding env. You can restore it using `pg_restore -O -C --dbname=`. You can also refer to -[ZkSync Node configuration management blueprint](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/00_quick_start.md#advanced-setup) +[ZKsync Node configuration management blueprint](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/external-node/00_quick_start.md#advanced-setup) for advanced DB instance configurations. [host_docker_internal](https://docs.docker.com/desktop/networking/#i-want-to-connect-from-a-container-to-a-service-on-the-host) ## Running -Assuming you have the zkSync node Docker image, an env file with the prepared configuration, and you have restored your +Assuming you have the ZKsync node Docker image, an env file with the prepared configuration, and you have restored your DB with the pg dump, that is all you need. Sample running command: @@ -69,9 +69,9 @@ in RocksDB (mainly the Merkle tree) is absent. Before the node can make any prog RocksDB and verify consistency. The exact time required for that depends on the hardware configuration, but it is reasonable to expect the state rebuild on the mainnet to take more than 20 hours. -## Redeploying the zkSync node with a new PG dump +## Redeploying the ZKsync node with a new PG dump -If you've been running the zkSync node for some time and are going to redeploy it using a new PG dump, you should +If you've been running the ZKsync node for some time and are going to redeploy it using a new PG dump, you should - Stop the EN - Remove SK cache (corresponding to `EN_STATE_CACHE_PATH`) diff --git a/docs/guides/external-node/04_observability.md b/docs/guides/external-node/04_observability.md index 1199503cc924..538c1130b62e 100644 --- a/docs/guides/external-node/04_observability.md +++ b/docs/guides/external-node/04_observability.md @@ -1,6 +1,6 @@ -# zkSync node Observability +# ZKsync node Observability -The zkSync node provides several options for setting up observability. Configuring logs and sentry is described in the +The ZKsync node provides several options for setting up observability. Configuring logs and sentry is described in the [configuration](./02_configuration.md) section, so this section focuses on the exposed metrics. This section is written with the assumption that you're familiar with @@ -16,7 +16,7 @@ By default, latency histograms are distributed in the following buckets (in seco ## Metrics -zkSync node exposes a lot of metrics, a significant amount of which aren't interesting outside the development flow. +ZKsync node exposes a lot of metrics, a significant amount of which aren't interesting outside the development flow. This section's purpose is to highlight metrics that may be worth observing in the external setup. If you are not planning to scrape Prometheus metrics, please unset `EN_PROMETHEUS_PORT` environment variable to prevent @@ -25,7 +25,7 @@ memory leaking. | Metric name | Type | Labels | Description | | ---------------------------------------------- | --------- | ------------------------------------- | ------------------------------------------------------------------ | | `external_node_synced` | Gauge | - | 1 if synced, 0 otherwise. Matches `eth_call` behavior | -| `external_node_sync_lag` | Gauge | - | How many blocks behind the main node the zkSync node is | +| `external_node_sync_lag` | Gauge | - | How many blocks behind the main node the ZKsync node is | | `external_node_fetcher_requests` | Histogram | `stage`, `actor` | Duration of requests performed by the different fetcher components | | `external_node_fetcher_cache_requests` | Histogram | - | Duration of requests performed by the fetcher cache layer | | `external_node_fetcher_miniblock` | Gauge | `status` | The number of the last L2 block update fetched from the main node | @@ -40,12 +40,12 @@ memory leaking. ## Interpretation -After applying a dump, the zkSync node has to rebuild the Merkle tree to verify the correctness of the state in +After applying a dump, the ZKsync node has to rebuild the Merkle tree to verify the correctness of the state in PostgreSQL. During this stage, `server_block_number { stage='tree_lightweight_mode' }` is increasing from 0 to -`server_block_number { stage='sealed' }`, while the latter does not increase (zkSync node needs the tree to be +`server_block_number { stage='sealed' }`, while the latter does not increase (ZKsync node needs the tree to be up-to-date to progress). -After that, the zkSync node has to sync with the main node. `server_block_number { stage='sealed' }` is increasing, and +After that, the ZKsync node has to sync with the main node. `server_block_number { stage='sealed' }` is increasing, and `external_node_sync_lag` is decreasing. Once the node is synchronized, it is indicated by the `external_node_synced`. diff --git a/docs/guides/external-node/05_troubleshooting.md b/docs/guides/external-node/05_troubleshooting.md index 1179a3e43efb..43d6ae26b135 100644 --- a/docs/guides/external-node/05_troubleshooting.md +++ b/docs/guides/external-node/05_troubleshooting.md @@ -1,6 +1,6 @@ -# zkSync node Troubleshooting +# ZKsync node Troubleshooting -The zkSync node tries to follow the fail-fast principle: if an anomaly is discovered, instead of attempting state +The ZKsync node tries to follow the fail-fast principle: if an anomaly is discovered, instead of attempting state recovery, in most cases it will restart. Most of the time it will manifest as crashes, and if it happens once, it shouldn't be treated as a problem. @@ -24,8 +24,8 @@ Other kinds of panic aren't normally expected. While in most cases, the state wi ## Genesis Issues -The zkSync node is supposed to start with an applied DB dump. If you see any genesis-related errors, it probably means -the zkSync node was started without an applied dump. +The ZKsync node is supposed to start with an applied DB dump. If you see any genesis-related errors, it probably means +the ZKsync node was started without an applied dump. [contact_us]: https://zksync.io/contact @@ -43,7 +43,7 @@ you don't consider actionable, you may disable logs for a component by tweaking | WARN | "Following transport error occurred" | There was a problem with fetching data from the main node. | | WARN | "Unable to get the gas price" | There was a problem with fetching data from the main node. | | WARN | "Consistency checker error" | There are problems querying L1, check the Web3 URL you specified in the config. | -| WARN | "Reorg detected" | Reorg was detected on the main node, the zkSync node will rollback and restart | +| WARN | "Reorg detected" | Reorg was detected on the main node, the ZKsync node will rollback and restart | Same as with panics, normally it's only a problem if a WARN+ level log appears many times in a row. diff --git a/docs/guides/external-node/06_components.md b/docs/guides/external-node/06_components.md index 2210842c9d1a..733400058a82 100644 --- a/docs/guides/external-node/06_components.md +++ b/docs/guides/external-node/06_components.md @@ -1,29 +1,29 @@ -# zkSync node components +# ZKsync node components This section contains an overview of the EN's main components. ## API -The zkSync node can serve both the HTTP and the WS Web3 API, as well as PubSub. Whenever possible, it provides data +The ZKsync node can serve both the HTTP and the WS Web3 API, as well as PubSub. Whenever possible, it provides data based on the local state, with a few exceptions: - Submitting transactions: Since it is a read replica, submitted transactions are proxied to the main node, and the response is returned from the main node. -- Querying transactions: The zkSync node is not aware of the main node's mempool, and it does not sync rejected - transactions. Therefore, if a local lookup for a transaction or its receipt fails, the zkSync node will attempt the +- Querying transactions: The ZKsync node is not aware of the main node's mempool, and it does not sync rejected + transactions. Therefore, if a local lookup for a transaction or its receipt fails, the ZKsync node will attempt the same query on the main node. Apart from these cases, the API does not depend on the main node. Even if the main node is temporarily unavailable, the -zkSync node can continue to serve the state it has locally. +ZKsync node can continue to serve the state it has locally. ## Fetcher -The Fetcher component is responsible for maintaining synchronization between the zkSync node and the main node. Its +The Fetcher component is responsible for maintaining synchronization between the ZKsync node and the main node. Its primary task is to fetch new blocks in order to update the local chain state. However, its responsibilities extend beyond that. For instance, the Fetcher is also responsible for keeping track of L1 batch statuses. This involves monitoring whether locally applied batches have been committed, proven, or executed on L1. -It is worth noting that in addition to fetching the _state_, the zkSync node also retrieves the L1 gas price from the +It is worth noting that in addition to fetching the _state_, the ZKsync node also retrieves the L1 gas price from the main node for the purpose of estimating fees for L2 transactions (since this also happens based on the local state). This information is necessary to ensure that gas estimations are performed in the exact same manner as the main node, thereby reducing the chances of a transaction not being included in a block. @@ -32,23 +32,23 @@ thereby reducing the chances of a transaction not being included in a block. The State Keeper component serves as the "sequencer" part of the node. It shares most of its functionality with the main node, with one key distinction. The main node retrieves transactions from the mempool and has the authority to decide -when a specific L2 block or L1 batch should be sealed. On the other hand, the zkSync node retrieves transactions from +when a specific L2 block or L1 batch should be sealed. On the other hand, the ZKsync node retrieves transactions from the queue populated by the Fetcher and seals the corresponding blocks/batches based on the data obtained from the Fetcher queue. -The actual execution of batches takes place within the VM, which is identical in both the Main and zkSync nodes. +The actual execution of batches takes place within the VM, which is identical in both the Main and ZKsync nodes. ## Reorg Detector -In zkSync Era, it is theoretically possible for L1 batches to be reverted before the corresponding "execute" operation +In ZKsync Era, it is theoretically possible for L1 batches to be reverted before the corresponding "execute" operation is applied on L1, that is before the block is [final][finality]. Such situations are highly uncommon and typically occur due to significant issues: e.g. a bug in the sequencer implementation preventing L1 batch commitment. Prior to batch -finality, the zkSync operator can perform a rollback, reverting one or more batches and restoring the blockchain state +finality, the ZKsync operator can perform a rollback, reverting one or more batches and restoring the blockchain state to a previous point. Finalized batches cannot be reverted at all. -However, even though such situations are rare, the zkSync node must handle them correctly. +However, even though such situations are rare, the ZKsync node must handle them correctly. -To address this, the zkSync node incorporates a Reorg Detector component. This module keeps track of all L1 batches that +To address this, the ZKsync node incorporates a Reorg Detector component. This module keeps track of all L1 batches that have not yet been finalized. It compares the locally obtained state root hashes with those provided by the main node's API. If the root hashes for the latest available L1 batch do not match, the Reorg Detector searches for the specific L1 batch responsible for the divergence. Subsequently, it rolls back the local state and restarts the node. Upon restart, @@ -67,13 +67,13 @@ When the Consistency Checker detects that a particular batch has been sent to L1 known as the "block commitment" for the L1 transaction. The block commitment contains crucial data such as the state root and batch number, and is the same commitment that is used for generating a proof for the batch. The Consistency Checker then compares the locally obtained commitment with the actual commitment sent to L1. If the data does not match, -it indicates a potential bug in either the main node or zkSync node implementation or that the main node API has -provided incorrect data. In either case, the state of the zkSync node cannot be trusted, and the zkSync node enters a +it indicates a potential bug in either the main node or ZKsync node implementation or that the main node API has +provided incorrect data. In either case, the state of the ZKsync node cannot be trusted, and the ZKsync node enters a crash loop until the issue is resolved. ## Health check server -The zkSync node also exposes an additional server that returns HTTP 200 response when the zkSync node is operating -normally, and HTTP 503 response when some of the health checks don't pass (e.g. when the zkSync node is not fully +The ZKsync node also exposes an additional server that returns HTTP 200 response when the ZKsync node is operating +normally, and HTTP 503 response when some of the health checks don't pass (e.g. when the ZKsync node is not fully initialized yet). This server can be used, for example, to implement the readiness probe in an orchestration solution you use. diff --git a/docs/guides/external-node/prepared_configs/mainnet-config.env b/docs/guides/external-node/prepared_configs/mainnet-config.env index efd087b0bb32..35278205b96f 100644 --- a/docs/guides/external-node/prepared_configs/mainnet-config.env +++ b/docs/guides/external-node/prepared_configs/mainnet-config.env @@ -75,7 +75,7 @@ RUST_LIB_BACKTRACE=1 # -------------- THE FOLLOWING VARIABLES DEPEND ON THE ENV --------------- # ------------------------------------------------------------------------ -# URL of the main zkSync node. +# URL of the main ZKsync node. EN_MAIN_NODE_URL=https://zksync2-mainnet.zksync.io EN_L2_CHAIN_ID=324 diff --git a/docs/guides/external-node/prepared_configs/testnet-goerli-config-deprecated.env b/docs/guides/external-node/prepared_configs/testnet-goerli-config-deprecated.env index 2c1723460a27..eb8b6481d75f 100644 --- a/docs/guides/external-node/prepared_configs/testnet-goerli-config-deprecated.env +++ b/docs/guides/external-node/prepared_configs/testnet-goerli-config-deprecated.env @@ -75,7 +75,7 @@ RUST_LIB_BACKTRACE=1 # -------------- THE FOLLOWING VARIABLES DEPEND ON THE ENV --------------- # ------------------------------------------------------------------------ -# URL of the main zkSync node. +# URL of the main ZKsync node. EN_MAIN_NODE_URL=https://zksync2-testnet.zksync.dev EN_L2_CHAIN_ID=280 diff --git a/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env b/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env index d85543a8ec56..98e2ee6bd510 100644 --- a/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env +++ b/docs/guides/external-node/prepared_configs/testnet-sepolia-config.env @@ -75,7 +75,7 @@ RUST_LIB_BACKTRACE=1 # -------------- THE FOLLOWING VARIABLES DEPEND ON THE ENV --------------- # ------------------------------------------------------------------------ -# URL of the main zkSync node. +# URL of the main ZKsync node. EN_MAIN_NODE_URL=https://sepolia.era.zksync.dev EN_L2_CHAIN_ID=300 diff --git a/docs/guides/launch.md b/docs/guides/launch.md index 2889216dbbe9..35588debd3ae 100644 --- a/docs/guides/launch.md +++ b/docs/guides/launch.md @@ -1,6 +1,6 @@ # Running the application -This document covers common scenarios for launching zkSync applications set locally. +This document covers common scenarios for launching ZKsync applications set locally. ## Prerequisites diff --git a/docs/guides/repositories.md b/docs/guides/repositories.md index d43bab72e5e2..36a52a2ae76f 100644 --- a/docs/guides/repositories.md +++ b/docs/guides/repositories.md @@ -1,6 +1,6 @@ # Repositories -## zkSync +## ZKsync ### Core components @@ -54,10 +54,10 @@ | --------------------------------------------------------------- | ----------------------------------------------------------------------------- | | [era-test-node](https://github.com/matter-labs/era-test-node) | In memory node for development and smart contract debugging | | [local-setup](https://github.com/matter-labs/local-setup) | Docker-based zk server (together with L1), that can be used for local testing | -| [zksync-cli](https://github.com/matter-labs/zksync-cli) | Command line tool to interact with zksync | -| [block-explorer](https://github.com/matter-labs/block-explorer) | Online blockchain browser for viewing and analyzing zkSync chain | -| [dapp-portal](https://github.com/matter-labs/dapp-portal) | zkSync Wallet + Bridge DApp | -| [hardhat-zksync](https://github.com/matter-labs/hardhat-zksync) | zkSync Hardhat plugins | +| [zksync-cli](https://github.com/matter-labs/zksync-cli) | Command line tool to interact with ZKsync | +| [block-explorer](https://github.com/matter-labs/block-explorer) | Online blockchain browser for viewing and analyzing ZKsync chain | +| [dapp-portal](https://github.com/matter-labs/dapp-portal) | ZKsync Wallet + Bridge DApp | +| [hardhat-zksync](https://github.com/matter-labs/hardhat-zksync) | ZKsync Hardhat plugins | | [zksolc-bin](https://github.com/matter-labs/zksolc-bin) | solc compiler binaries | | [zkvyper-bin](https://github.com/matter-labs/zkvyper-bin) | vyper compiler binaries | @@ -65,16 +65,16 @@ | Public repository | Description | | --------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------- | -| [zksync-web-era-docs](https://github.com/matter-labs/zksync-docs) | [Public zkSync documentation](https://docs.zksync.io), API descriptions etc. | +| [zksync-web-era-docs](https://github.com/matter-labs/zksync-docs) | [Public ZKsync documentation](https://docs.zksync.io), API descriptions etc. | | [zksync-contract-templates](https://github.com/matter-labs/zksync-contract-templates) | Quick contract deployment and testing with tools like Hardhat on Solidity or Vyper | | [zksync-frontend-templates](https://github.com/matter-labs/zksync-frontend-templates) | Rapid UI development with templates for Vue, React, Next.js, Nuxt, Vite, etc. | -| [zksync-scripting-templates](https://github.com/matter-labs/zksync-scripting-templates) | Automated interactions and advanced zkSync operations using Node.js | -| [tutorials](https://github.com/matter-labs/tutorials) | Tutorials for developing on zkSync | +| [zksync-scripting-templates](https://github.com/matter-labs/zksync-scripting-templates) | Automated interactions and advanced ZKsync operations using Node.js | +| [tutorials](https://github.com/matter-labs/tutorials) | Tutorials for developing on ZKsync | -## zkSync Lite +## ZKsync Lite | Public repository | Description | | --------------------------------------------------------------------------- | -------------------------------- | -| [zksync](https://github.com/matter-labs/zksync) | zkSync Lite implementation | -| [zksync-lite-docs](https://github.com/matter-labs/zksync-lite-docs) | Public zkSync Lite documentation | +| [zksync](https://github.com/matter-labs/zksync) | ZKsync Lite implementation | +| [ZKsync-lite-docs](https://github.com/matter-labs/zksync-lite-docs) | Public ZKsync Lite documentation | | [zksync-dapp-checkout](https://github.com/matter-labs/zksync-dapp-checkout) | Batch payments DApp | diff --git a/docs/guides/setup-dev.md b/docs/guides/setup-dev.md index b8db0c1575c7..4e005fc2795f 100644 --- a/docs/guides/setup-dev.md +++ b/docs/guides/setup-dev.md @@ -35,7 +35,7 @@ foundryup --branch master ## Supported operating systems -zkSync currently can be launched on any \*nix operating system (e.g. any linux distribution or MacOS). +ZKsync currently can be launched on any \*nix operating system (e.g. any linux distribution or MacOS). If you're using Windows, then make sure to use WSL 2, since WSL 1 is known to cause troubles. @@ -43,7 +43,7 @@ Additionally, if you are going to use WSL 2, make sure that your project is loca accessing NTFS partitions from within WSL is very slow. If you're using MacOS with an ARM processor (e.g. M1/M2), make sure that you are working in the _native_ environment -(e.g. your terminal and IDE don't run in Rosetta, and your toolchain is native). Trying to work with zkSync code via +(e.g. your terminal and IDE don't run in Rosetta, and your toolchain is native). Trying to work with ZKsync code via Rosetta may cause problems that are hard to spot and debug, so make sure to check everything before you start. If you are a NixOS user or would like to have a reproducible environment, skip to the section about `nix`. diff --git a/docs/specs/blocks_batches.md b/docs/specs/blocks_batches.md index ce678edf937d..c5d846a39736 100644 --- a/docs/specs/blocks_batches.md +++ b/docs/specs/blocks_batches.md @@ -196,7 +196,7 @@ The hash of an L2 block is To add a transaction hash to the current miniblock we use the `appendTransactionToCurrentL2Block` [function](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/system-contracts/contracts/SystemContext.sol#L373). -Since zkSync is a state-diff based rollup, there is no way to deduce the hashes of the L2 blocks based on the +Since ZKsync is a state-diff based rollup, there is no way to deduce the hashes of the L2 blocks based on the transactions’ in the batch (because there is no access to the transaction’s hashes). At the same time, in order to server `blockhash` method, the VM requires the knowledge of some of the previous L2 block hashes. In order to save up on pubdata (by making sure that the same storage slots are reused, i.e. we only have repeated writes) we diff --git a/docs/specs/data_availability/pubdata.md b/docs/specs/data_availability/pubdata.md index 3584a0430557..0bbb753411c1 100644 --- a/docs/specs/data_availability/pubdata.md +++ b/docs/specs/data_availability/pubdata.md @@ -1,6 +1,6 @@ # Handling pubdata in Boojum -Pubdata in zkSync can be divided up into 4 different categories: +Pubdata in ZKsync can be divided up into 4 different categories: 1. L2 to L1 Logs 2. L2 to L1 Messages @@ -13,7 +13,7 @@ pre-Boojum system these are represented as separate fields while for boojum they array. Once 4844 gets integrated this bytes array will move from being part of the calldata to blob data. While the structure of the pubdata changes, the way in which one can go about pulling the information will remain the -same. Basically, we just need to filter all of the transactions to the L1 zkSync contract for only the `commitBatches` +same. Basically, we just need to filter all of the transactions to the L1 ZKsync contract for only the `commitBatches` transactions where the proposed block has been referenced by a corresponding `executeBatches` call (the reason for this is that a committed or even proven block can be reverted but an executed one cannot). Once we have all the committed batches that have been executed, we then will pull the transaction input and the relevant fields, applying them in order @@ -106,7 +106,7 @@ be [applied](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/system-contracts/contracts/L1Messenger.sol#L110): `chainedLogsHash = keccak256(chainedLogsHash, hashedLog)`. L2→L1 logs have the same 88-byte format as in the current -version of zkSync. +version of ZKsync. Note, that the user is charged for necessary future the computation that will be needed to calculate the final merkle root. It is roughly 4x higher than the cost to calculate the hash of the leaf, since the eventual tree might have be 4x @@ -179,7 +179,7 @@ With Boojum, `factoryDeps` are included within the `totalPubdata` bytes and have ### Compressed Bytecode Publishing -This part stays the same in a pre and post boojum zkSync. Unlike uncompressed bytecode which are published as part of +This part stays the same in a pre and post boojum ZKsync. Unlike uncompressed bytecode which are published as part of `factoryDeps`, compressed bytecodes are published as long l2 → l1 messages which can be seen [here](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/system-contracts/contracts/Compressor.sol#L80). @@ -254,7 +254,7 @@ markAsPublished(hash(_bytecode)) ## Storage diff publishing -zkSync is a statediff-based rollup and so publishing the correct state diffs plays an integral role in ensuring data +ZKsync is a statediff-based rollup and so publishing the correct state diffs plays an integral role in ensuring data availability. ### How publishing of storage diffs worked before Boojum @@ -287,9 +287,9 @@ These two fields would be then included into the block commitment and checked by ### Difference between initial and repeated writes -zkSync publishes state changes that happened within the batch instead of transactions themselves. Meaning, that for +ZKsync publishes state changes that happened within the batch instead of transactions themselves. Meaning, that for instance some storage slot `S` under account `A` has changed to value `V`, we could publish a triple of `A,S,V`. Users -by observing all the triples could restore the state of zkSync. However, note that our tree unlike Ethereum’s one is not +by observing all the triples could restore the state of ZKsync. However, note that our tree unlike Ethereum’s one is not account based (i.e. there is no first layer of depth 160 of the merkle tree corresponding to accounts and second layer of depth 256 of the merkle tree corresponding to users). Our tree is “flat”, i.e. a slot `S` under account `A` is just stored in the leaf number `H(S,A)`. Our tree is of depth 256 + 8 (the 256 is for these hashed account/key pairs and 8 is diff --git a/docs/specs/l1_l2_communication/l1_to_l2.md b/docs/specs/l1_l2_communication/l1_to_l2.md index ed1605a039a6..f4a23219e277 100644 --- a/docs/specs/l1_l2_communication/l1_to_l2.md +++ b/docs/specs/l1_l2_communication/l1_to_l2.md @@ -1,6 +1,6 @@ # Handling L1→L2 ops -The transactions on zkSync can be initiated not only on L2, but also on L1. There are two types of transactions that can +The transactions on ZKsync can be initiated not only on L2, but also on L1. There are two types of transactions that can be initiated on L1: - Priority operations. These are the kind of operations that any user can create. @@ -103,7 +103,7 @@ We also remember that the upgrade transaction has been processed in this batch ( ### Revert -In a very rare event when the team needs to revert the batch with the upgrade on zkSync, the +In a very rare event when the team needs to revert the batch with the upgrade on ZKsync, the `l2SystemContractsUpgradeBatchNumber` is [reset](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/contracts/ethereum/contracts/zksync/facets/Executor.sol#L412). diff --git a/docs/specs/l1_smart_contracts.md b/docs/specs/l1_smart_contracts.md index 207920476602..65c408714ba3 100644 --- a/docs/specs/l1_smart_contracts.md +++ b/docs/specs/l1_smart_contracts.md @@ -235,8 +235,8 @@ The diagram below outlines the complete journey from the initiation of an operat ## ValidatorTimelock -An intermediate smart contract between the validator EOA account and the zkSync smart contract. Its primary purpose is -to provide a trustless means of delaying batch execution without modifying the main zkSync contract. zkSync actively +An intermediate smart contract between the validator EOA account and the ZKsync smart contract. Its primary purpose is +to provide a trustless means of delaying batch execution without modifying the main ZKsync contract. ZKsync actively monitors the chain activity and reacts to any suspicious activity by freezing the chain. This allows time for investigation and mitigation before resuming normal operations. @@ -246,12 +246,12 @@ the Alpha stage. This contract consists of four main functions `commitBatches`, `proveBatches`, `executeBatches`, and `revertBatches`, which can be called only by the validator. -When the validator calls `commitBatches`, the same calldata will be propagated to the zkSync contract (`DiamondProxy` +When the validator calls `commitBatches`, the same calldata will be propagated to the ZKsync contract (`DiamondProxy` through `call` where it invokes the `ExecutorFacet` through `delegatecall`), and also a timestamp is assigned to these batches to track the time these batches are committed by the validator to enforce a delay between committing and execution of batches. Then, the validator can prove the already committed batches regardless of the mentioned timestamp, -and again the same calldata (related to the `proveBatches` function) will be propagated to the zkSync contract. After -the `delay` is elapsed, the validator is allowed to call `executeBatches` to propagate the same calldata to zkSync +and again the same calldata (related to the `proveBatches` function) will be propagated to the ZKsync contract. After +the `delay` is elapsed, the validator is allowed to call `executeBatches` to propagate the same calldata to ZKsync contract. The owner of the ValidatorTimelock contract is the same as the owner of the Governance contract - Matter Labs multisig. diff --git a/docs/specs/prover/overview.md b/docs/specs/prover/overview.md index a7f814a458a8..5ac6dd59b772 100644 --- a/docs/specs/prover/overview.md +++ b/docs/specs/prover/overview.md @@ -1,4 +1,4 @@ -# Intro to zkSync’s ZK +# Intro to ZKsync’s ZK This page is specific to our cryptography. For a general introduction, please read: [https://docs.zksync.io/build/developer-reference/rollups.html](https://docs.zksync.io/build/developer-reference/rollups.html) @@ -6,8 +6,8 @@ This page is specific to our cryptography. For a general introduction, please re As a ZK rollup, we want everything to be verified by cryptography and secured by Ethereum. The power of ZK allows for transaction compression, reducing fees for users while inheriting the same security. -ZK Proofs allow a verifier to easily check whether a prover has done a computation correctly. For zkSync, the prover -will prove the correct execution of zkSync’s EVM, and a smart contract on Ethereum will verify the proof is correct. +ZK Proofs allow a verifier to easily check whether a prover has done a computation correctly. For ZKsync, the prover +will prove the correct execution of ZKsync’s EVM, and a smart contract on Ethereum will verify the proof is correct. In more detail, there are several steps. @@ -46,7 +46,7 @@ It is very important that every step is actually “constrained”. The prover m If the circuit is missing a constraint, then a malicious prover can create proofs that will pass verification but not be valid. The ZK terminology for this is that an underconstrained circuit could lead to a soundness error. -### What do zkSync’s circuits prove +### What do ZKsync’s circuits prove The main goal of our circuits is to prove correct execution of our VM. This includes proving each opcode run within the VM, as well as other components such as precompiles, storage, and circuits that connect everything else together. This diff --git a/docs/specs/prover/zk_terminology.md b/docs/specs/prover/zk_terminology.md index a0b7d101a64f..a747bb962998 100644 --- a/docs/specs/prover/zk_terminology.md +++ b/docs/specs/prover/zk_terminology.md @@ -20,14 +20,14 @@ revealing the actual information. ### Constraint -A constraint is a rule or restriction that a specific operation or set of operations must follow. zkSync uses +A constraint is a rule or restriction that a specific operation or set of operations must follow. ZKsync uses constraints to verify the validity of certain operations, and in the generation of proofs. Constraints can be missing, causing bugs, or there could be too many constraints, leading to restricted operations. ### Constraint degree The "constraint degree" of a constraint system refers to the maximum degree of the polynomial gates in the system. In -simpler terms, it’s the highest power of polynomial equations of the constraint system. At zkSync, we allow gates with +simpler terms, it’s the highest power of polynomial equations of the constraint system. At ZKsync, we allow gates with degree 8 or lower. ### Constraint system @@ -42,7 +42,7 @@ assignment of values to these Variables, ensuring that the rules still hold true The geometry defines the number of rows and columns in the constraint system. As part of PLONK arithmetization, the witness data is arranged into a grid, where each row defines a gate (or a few gates), and the columns are as long as -needed to hold all of the witness data. At zkSync, we have ~164 base witness columns. +needed to hold all of the witness data. At ZKsync, we have ~164 base witness columns. ### Log @@ -64,9 +64,9 @@ prover to the verifier. ### Prover -In our zkSync zk-rollup context, the prover is used to process a set of transactions executing smart contracts in a +In our ZKsync zk-rollup context, the prover is used to process a set of transactions executing smart contracts in a succinct and efficient manner. It computes proofs that all the transactions are correct and ensures a valid transition -from one state to another. The proof will be sent to a Verifier smart contract on Ethereum. At zkSync, we prove state +from one state to another. The proof will be sent to a Verifier smart contract on Ethereum. At ZKsync, we prove state diffs of a block of transactions, in order to prove the new state root state is valid. ### Satisfiable diff --git a/docs/specs/zk_evm/account_abstraction.md b/docs/specs/zk_evm/account_abstraction.md index c106fafc880f..0ea2e3fa4a09 100644 --- a/docs/specs/zk_evm/account_abstraction.md +++ b/docs/specs/zk_evm/account_abstraction.md @@ -1,6 +1,6 @@ # Account abstraction -One of the other important features of zkSync is the support of account abstraction. It is highly recommended to read +One of the other important features of ZKsync is the support of account abstraction. It is highly recommended to read the documentation on our AA protocol here: [https://docs.zksync.io/build/developer-reference/account-abstraction](https://docs.zksync.io/build/developer-reference/account-abstraction) diff --git a/docs/specs/zk_evm/bootloader.md b/docs/specs/zk_evm/bootloader.md index ec7f8378151d..41dfefa85168 100644 --- a/docs/specs/zk_evm/bootloader.md +++ b/docs/specs/zk_evm/bootloader.md @@ -6,7 +6,7 @@ On standard Ethereum clients, the workflow for executing blocks is the following 2. Gather the state changes (if the transaction has not reverted), apply them to the state. 3. Go back to step (1) if the block gas limit has not been yet exceeded. -However, having such flow on zkSync (i.e. processing transaction one-by-one) would be too inefficient, since we have to +However, having such flow on ZKsync (i.e. processing transaction one-by-one) would be too inefficient, since we have to run the entire proving workflow for each individual transaction. That’s why we need the _bootloader_: instead of running N transactions separately, we run the entire batch (set of blocks, more can be found [here](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/Smart%20contract%20Section/Batches%20%26%20L2%20blocks%20on%20zkSync.md)) @@ -19,7 +19,7 @@ unlike system contracts, the bootloader’s code is not stored anywhere on L2. T bootloader’s address as formal. It only exists for the sake of providing some value to `this` / `msg.sender`/etc. When someone calls the bootloader address (e.g. to pay fees) the EmptyContract’s code is actually invoked. -Bootloader is the program that accepts an array of transactions and executes the entire zkSync batch. This section will +Bootloader is the program that accepts an array of transactions and executes the entire ZKsync batch. This section will expand on its invariants and methods. ## Playground bootloader vs proved bootloader @@ -62,12 +62,12 @@ supported: - Note, that unlike type 1 and type 2 transactions, `reserved0` field can be set to a non-zero value, denoting that this legacy transaction is EIP-155-compatible and its RLP encoding (as well as signature) should contain the `chainId` of the system. -- `txType`: 1. It means that the transaction is of type 1, i.e. transactions access list. zkSync does not support access +- `txType`: 1. It means that the transaction is of type 1, i.e. transactions access list. ZKsync does not support access lists in any way, so no benefits of fulfilling this list will be provided. The access list is assumed to be empty. The same restrictions as for type 0 are enforced, but also `reserved0` must be 0. - `txType`: 2. It is EIP1559 transactions. The same restrictions as for type 1 apply, but now `maxFeePerErgs` may not be equal to `getMaxPriorityFeePerErg`. -- `txType`: 113. It is zkSync transaction type. This transaction type is intended for AA support. The only restriction +- `txType`: 113. It is ZKsync transaction type. This transaction type is intended for AA support. The only restriction that applies to this transaction type: fields `reserved0..reserved4` must be equal to 0. - `txType`: 254. It is a transaction type that is used for upgrading the L2 system. This is the only type of transaction is allowed to start a transaction out of the name of the contracts in kernel space. @@ -238,7 +238,7 @@ succeeded, the slot `2^19 - 1024 + i` will be marked as 1 and 0 otherwise. ## L2 transactions -On zkSync, every address is a contract. Users can start transactions from their EOA accounts, because every address that +On ZKsync, every address is a contract. Users can start transactions from their EOA accounts, because every address that does not have any contract deployed on it implicitly contains the code defined in the [DefaultAccount.sol](https://github.com/code-423n4/2023-10-zksync/blob/main/code/system-contracts/contracts/DefaultAccount.sol) file. Whenever anyone calls a contract that is not in kernel space (i.e. the address is ≥ 2^16) and does not have any @@ -323,7 +323,7 @@ Also, we [set](https://github.com/code-423n4/2023-10-zksync/blob/ef99273a8fdb19f5912ca38ba46d6bd02071363d/code/system-contracts/bootloader/bootloader.yul#L3812) the fictive L2 block’s data. Then, we call the system context to ensure that it publishes the timestamp of the L2 block as well as L1 batch. We also reset the `txNumberInBlock` counter to avoid its state diffs from being published on L1. -You can read more about block processing on zkSync +You can read more about block processing on ZKsync [here](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/Smart%20contract%20Section/Batches%20&%20L2%20blocks%20on%20zkSync.md). After that, we publish the hash as well as the number of priority operations in this batch. More on it diff --git a/docs/specs/zk_evm/fee_model.md b/docs/specs/zk_evm/fee_model.md index a75d45a737b9..78f9d38ae366 100644 --- a/docs/specs/zk_evm/fee_model.md +++ b/docs/specs/zk_evm/fee_model.md @@ -1,4 +1,4 @@ -# zkSync fee model +# ZKsync fee model This document will assume that you already know how gas & fees work on Ethereum. @@ -6,18 +6,18 @@ On Ethereum, all the computational, as well as storage costs, are represented vi certain amount of gas, which is generally constant (though it may change during [upgrades](https://blog.ethereum.org/2021/03/08/ethereum-berlin-upgrade-announcement)). -zkSync as well as other L2s have the issue which does not allow to adopt the same model as the one for Ethereum so +ZKsync as well as other L2s have the issue which does not allow to adopt the same model as the one for Ethereum so easily: the main reason is the requirement for publishing of the pubdata on Ethereum. This means that prices for L2 transactions will depend on the volatile L1 gas prices and can not be simply hardcoded. ## High-level description -zkSync, being a zkRollup is required to prove every operation with zero knowledge proofs. That comes with a few nuances. +ZKsync, being a zkRollup is required to prove every operation with zero knowledge proofs. That comes with a few nuances. ### `gas_per_pubdata_limit` -As already mentioned, the transactions on zkSync depend on volatile L1 gas costs to publish the pubdata for batch, -verify proofs, etc. For this reason, zkSync-specific EIP712 transactions contain the `gas_per_pubdata_limit` field in +As already mentioned, the transactions on ZKsync depend on volatile L1 gas costs to publish the pubdata for batch, +verify proofs, etc. For this reason, ZKsync-specific EIP712 transactions contain the `gas_per_pubdata_limit` field in them, denoting the maximum price in _gas_ that the operator \*\*can charge from users for a single byte of pubdata. For Ethereum transactions (which do not contain this field), it is enforced that the operator will not use a value @@ -28,23 +28,23 @@ larger value than a certain constant. The operations tend to have different “complexity”/”pricing” in zero knowledge proof terms than in standard CPU terms. For instance, `keccak256` which was optimized for CPU performance, will cost more to prove. -That’s why you will find the prices for operations on zkSync a lot different from the ones on Ethereum. +That’s why you will find the prices for operations on ZKsync a lot different from the ones on Ethereum. ### Different intrinsic costs Unlike Ethereum, where the intrinsic cost of transactions (`21000` gas) is used to cover the price of updating the -balances of the users, the nonce and signature verification, on zkSync these prices are _not_ included in the intrinsic +balances of the users, the nonce and signature verification, on ZKsync these prices are _not_ included in the intrinsic costs for transactions, due to the native support of account abstraction, meaning that each account type may have their own transaction cost. In theory, some may even use more zk-friendly signature schemes or other kinds of optimizations to allow cheaper transactions for their users. -That being said, zkSync transactions do come with some small intrinsic costs, but they are mostly used to cover costs +That being said, ZKsync transactions do come with some small intrinsic costs, but they are mostly used to cover costs related to the processing of the transaction by the bootloader which can not be easily measured in code in real-time. These are measured via testing and are hard coded. ### Batch overhead & limited resources of the batch -In order to process the batch, the zkSync team has to pay for proving of the batch, committing to it, etc. Processing a +In order to process the batch, the ZKsync team has to pay for proving of the batch, committing to it, etc. Processing a batch involves some operational costs as well. All of these values we call “Batch overhead”. It consists of two parts: - The L2 requirements for proving the circuits (denoted in L2 gas). @@ -57,7 +57,7 @@ resources_. While on Ethereum, the main reason for the existence of batch gas limit is to keep the system decentralized & load low, i.e. assuming the existence of the correct hardware, only time would be a requirement for a batch to adhere to. In the -case of zkSync batches, there are some limited resources the batch should manage: +case of ZKsync batches, there are some limited resources the batch should manage: - **Time.** The same as on Ethereum, the batch should generally not take too much time to be closed in order to provide better UX. To represent the time needed we use a batch gas limit, note that it is higher than the gas limit for a @@ -71,7 +71,7 @@ case of zkSync batches, there are some limited resources the batch should manage single slot happening in the same batch need to be published only once, we need to publish all the batch’s public data only after the transaction has been processed. Right now, we publish all the data with the storage diffs as well as L2→L1 messages, etc in a single transaction at the end of the batch. Most nodes have limit of 128kb per transaction - and so this is the limit that each zkSync batch should adhere to. + and so this is the limit that each ZKsync batch should adhere to. Each transaction spends the batch overhead proportionally to how close it consumes the resources above. @@ -79,7 +79,7 @@ Note, that before the transaction is executed, the system can not know how many transaction will actually take, so we need to charge for the worst case and provide the refund at the end of the transaction. -### How `baseFee` works on zkSync +### How `baseFee` works on ZKsync In order to protect us from DDoS attacks we need to set a limited `MAX_TRANSACTION_GAS_LIMIT` per transaction. Since the computation costs are relatively constant for us, we _could_ use a “fair” `baseFee` equal to the real costs for us to @@ -114,16 +114,16 @@ sure that the excess gas will be spent on the pubdata). ### High-level: conclusion -The zkSync fee model is meant to be the basis of the long-term fee model, which provides both robustness and security. +The ZKsync fee model is meant to be the basis of the long-term fee model, which provides both robustness and security. One of the most distinctive parts of it is the existing of the batch overhead, which is proportional for the resources consumed by the transaction. -The other distinctive feature of the fee model used on zkSync is the abundance of refunds, i.e.: +The other distinctive feature of the fee model used on ZKsync is the abundance of refunds, i.e.: - For unused limited system resources. - For overpaid computation. -This is needed because of the relatively big upfront payments required in zkSync to provide DDoS security. +This is needed because of the relatively big upfront payments required in ZKsync to provide DDoS security. ## Formalization @@ -156,7 +156,7 @@ contain almost any arbitrary value depending on the capacity of batch that we wa `BOOTLOADER_MEMORY_FOR_TXS` (_BM_) — The size of the bootloader memory that is used for transaction encoding (i.e. excluding the constant space, preallocated for other purposes). -`GUARANTEED_PUBDATA_PER_TX` (_PG_) — The guaranteed number of pubdata that should be possible to pay for in one zkSync +`GUARANTEED_PUBDATA_PER_TX` (_PG_) — The guaranteed number of pubdata that should be possible to pay for in one ZKsync batch. This is a number that should be enough for most reasonable cases. #### Derived constants diff --git a/docs/specs/zk_evm/precompiles.md b/docs/specs/zk_evm/precompiles.md index 4874bcdf9404..c6adc00410bb 100644 --- a/docs/specs/zk_evm/precompiles.md +++ b/docs/specs/zk_evm/precompiles.md @@ -19,7 +19,7 @@ nor the instructions to put the parameters in memory. For Go-Ethereum, the code being run is written in Go, and the gas costs are defined in each precompile spec. -In the case of zkSync Era, ecAdd and ecMul precompiles are written as a smart contract for two reasons: +In the case of ZKsync Era, ecAdd and ecMul precompiles are written as a smart contract for two reasons: - zkEVM needs to be able to prove their execution (and at the moment it cannot do that if the code being run is executed outside the VM) @@ -36,7 +36,7 @@ The arithmetic is carried out with the field elements encoded in the Montgomery operating in the Montgomery form speeds up the computation but also because the native modular multiplication, which is carried out by Yul's `mulmod` opcode, is very inefficient. -Instructions set on zkSync and EVM are different, so the performance of the same Yul/Solidity code can be efficient on +Instructions set on ZKsync and EVM are different, so the performance of the same Yul/Solidity code can be efficient on EVM, but not on zkEVM and opposite. One such very inefficient command is `mulmod`. On EVM there is a native opcode that makes modulo multiplication and it diff --git a/docs/specs/zk_evm/system_contracts.md b/docs/specs/zk_evm/system_contracts.md index 136d2136cd9e..48f072435519 100644 --- a/docs/specs/zk_evm/system_contracts.md +++ b/docs/specs/zk_evm/system_contracts.md @@ -26,7 +26,7 @@ values are set on genesis explicitly. Notably, if in the future we want to upgra This contract is also responsible for ensuring validity and consistency of batches, L2 blocks and virtual blocks. The implementation itself is rather straightforward, but to better understand this contract, please take a look at the [page](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/Smart%20contract%20Section/Batches%20&%20L2%20blocks%20on%20zkSync.md) -about the block processing on zkSync. +about the block processing on ZKsync. ## AccountCodeStorage @@ -86,7 +86,7 @@ and returns `success=1`. ## SHA256 & Keccak256 -Note that, unlike Ethereum, keccak256 is a precompile (_not an opcode_) on zkSync. +Note that, unlike Ethereum, keccak256 is a precompile (_not an opcode_) on ZKsync. These system contracts act as wrappers for their respective crypto precompile implementations. They are expected to be used frequently, especially keccak256, since Solidity computes storage slots for mapping and dynamic arrays with its @@ -128,7 +128,7 @@ More information on the extraAbiParams can be read ## KnownCodeStorage -This contract is used to store whether a certain code hash is “known”, i.e. can be used to deploy contracts. On zkSync, +This contract is used to store whether a certain code hash is “known”, i.e. can be used to deploy contracts. On ZKsync, the L2 stores the contract’s code _hashes_ and not the codes themselves. Therefore, it must be part of the protocol to ensure that no contract with unknown bytecode (i.e. hash with an unknown preimage) is ever deployed. @@ -151,9 +151,9 @@ The KnownCodesStorage contract is also responsible for ensuring that all the “ ## ContractDeployer & ImmutableSimulator -`ContractDeployer` is a system contract responsible for deploying contracts on zkSync. It is better to understand how it -works in the context of how the contract deployment works on zkSync. Unlike Ethereum, where `create`/`create2` are -opcodes, on zkSync these are implemented by the compiler via calls to the ContractDeployer system contract. +`ContractDeployer` is a system contract responsible for deploying contracts on ZKsync. It is better to understand how it +works in the context of how the contract deployment works on ZKsync. Unlike Ethereum, where `create`/`create2` are +opcodes, on ZKsync these are implemented by the compiler via calls to the ContractDeployer system contract. For additional security, we also distinguish the deployment of normal contracts and accounts. That’s why the main methods that will be used by the user are `create`, `create2`, `createAccount`, `create2Account`, which simulate the @@ -168,7 +168,7 @@ the L2 contract). Generally, rollups solve this issue in two ways: - XOR/ADD some kind of constant to addresses during L1→L2 communication. That’s how rollups closer to full EVM-equivalence solve it, since it allows them to maintain the same derivation rules on L1 at the expense of contract accounts on L1 having to redeploy on L2. -- Have different derivation rules from Ethereum. That is the path that zkSync has chosen, mainly because since we have +- Have different derivation rules from Ethereum. That is the path that ZKsync has chosen, mainly because since we have different bytecode than on EVM, CREATE2 address derivation would be different in practice anyway. You can see the rules for our address derivation in `getNewAddressCreate2`/ `getNewAddressCreate` methods in the @@ -179,7 +179,7 @@ way to support EVM bytecodes in the future. ### **Deployment nonce** -On Ethereum, the same nonce is used for CREATE for accounts and EOA wallets. On zkSync this is not the case, we use a +On Ethereum, the same nonce is used for CREATE for accounts and EOA wallets. On ZKsync this is not the case, we use a separate nonce called “deploymentNonce” to track the nonces for accounts. This was done mostly for consistency with custom accounts and for having multicalls feature in the future. @@ -197,13 +197,13 @@ custom accounts and for having multicalls feature in the future. - Calls `ImmutableSimulator` to set the immutables that are to be used for the deployed contract. Note how it is different from the EVM approach: on EVM when the contract is deployed, it executes the initCode and -returns the deployedCode. On zkSync, contracts only have the deployed code and can set immutables as storage variables +returns the deployedCode. On ZKsync, contracts only have the deployed code and can set immutables as storage variables returned by the constructor. ### **Constructor** On Ethereum, the constructor is only part of the initCode that gets executed during the deployment of the contract and -returns the deployment code of the contract. On zkSync, there is no separation between deployed code and constructor +returns the deployment code of the contract. On ZKsync, there is no separation between deployed code and constructor code. The constructor is always a part of the deployment code of the contract. In order to protect it from being called, the compiler-generated contracts invoke constructor only if the `isConstructor` flag provided (it is only available for the system contracts). You can read more about flags @@ -228,7 +228,7 @@ part of the compiler specification. This contract treats it simply as mapping fr address. Whenever a contract needs to access a value of some immutable, they call the -`ImmutableSimulator.getImmutable(getCodeAddress(), index)`. Note that on zkSync it is possible to get the current +`ImmutableSimulator.getImmutable(getCodeAddress(), index)`. Note that on ZKsync it is possible to get the current execution address you can read more about `getCodeAddress()` [here](https://github.com/matter-labs/zksync-era/blob/main/docs/guides/advanced/0_alternative_vm_intro.md#zkevm-specific-opcodes). @@ -248,7 +248,7 @@ are not in kernel space and have no contract deployed on them. This address: ## L1Messenger -A contract used for sending arbitrary length L2→L1 messages from zkSync to L1. While zkSync natively supports a rather +A contract used for sending arbitrary length L2→L1 messages from ZKsync to L1. While ZKsync natively supports a rather limited number of L1→L2 logs, which can transfer only roughly 64 bytes of data a time, we allowed sending nearly-arbitrary length L2→L1 messages with the following trick: diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/call.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/call.md index 71b40a0cb2ab..060ba8ec2346 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/call.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/call.md @@ -6,7 +6,7 @@ The call type is encoded on the assembly level, so we will describe the common h distinctions if there are any. For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#call-staticcall-delegatecall). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#call-staticcall-delegatecall). ## [CALL](https://www.evm.codes/#f1?fork=shanghai) diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/create.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/create.md index a35703545d62..eeecb93526ab 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/create.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/create.md @@ -3,7 +3,7 @@ The EVM CREATE instructions are handled similarly. For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#create-create2). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#create-create2). ## [CREATE](https://www.evm.codes/#f0?fork=shanghai) diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/return.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/return.md index 0e1756b6f193..014a2a3e47c1 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/return.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/evm/return.md @@ -12,7 +12,7 @@ is common for Yul and EVMLA representations. ## [RETURN](https://www.evm.codes/#f3?fork=shanghai) This instruction works differently in deploy code. For more information, see -[the zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#return-stop). +[the ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#return-stop). ### LLVM IR diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/evmla.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/evmla.md index 3304c2efe66c..06549962244b 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/evmla.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/evmla.md @@ -26,7 +26,7 @@ LLVM IR codegen references: The same as [setimmutable](yul.md#setimmutable). For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). LLVM IR codegen references: @@ -38,7 +38,7 @@ LLVM IR codegen references: The same as [loadimmutable](yul.md#loadimmutable). For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). LLVM IR codegen references: @@ -50,7 +50,7 @@ LLVM IR codegen references: The same as [linkersymbol](yul.md#linkersymbol). For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/libraries). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/libraries). [The LLVM IR generator code](https://github.com/matter-labs/era-compiler-solidity/blob/main/src/yul/parser/statement/expression/function_call/mod.rs#L956). diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/call.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/call.md index 9a6b39d54d7d..bada21b359d0 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/call.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/call.md @@ -1,4 +1,4 @@ -# zkSync Era Extension Simulation (call) +# ZKsync Era Extension Simulation (call) NOTES: diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/overview.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/overview.md index 889865e5f9be..a3fedb085c78 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/overview.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/overview.md @@ -1,6 +1,6 @@ -# zkSync Era Extensions +# ZKsync Era Extensions -Since we have no control over the Solidity compiler, we are using temporary hacks to support zkSync-specific +Since we have no control over the Solidity compiler, we are using temporary hacks to support ZKsync-specific instructions: - [Call substitutions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/call.md) diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/verbatim.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/verbatim.md index f2b1be0ff4f6..7291e5bf46a1 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/verbatim.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/extensions/verbatim.md @@ -1,4 +1,4 @@ -# zkSync Era Extension Simulation (verbatim) +# ZKsync Era Extension Simulation (verbatim) NOTES: diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/overview.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/overview.md index bc8daf3b2e1a..56515c9eadce 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/overview.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/overview.md @@ -5,7 +5,7 @@ In this specification, instructions are grouped by their relevance to the EVM in - [Native EVM instructions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/overview.md). - [Yul auxiliary instructions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/yul.md). - [EVM legacy assembly auxiliary instructions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/evmla.md). -- [zkSync Era extensions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/overview.md). +- [ZKsync Era extensions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/overview.md). Most of the EVM native instructions are represented in both Yul and EVM legacy assembly IRs. If they are not, it is stated explicitly in the description of each instruction. @@ -25,7 +25,7 @@ Every instruction is translated via two IRs available in the Solidity compiler u ## Yul Extensions -At the moment there is no way of adding zkSync-specific instructions to Yul as long as we use the official Solidity +At the moment there is no way of adding ZKsync-specific instructions to Yul as long as we use the official Solidity compiler, which would produce an error on an unknown instruction. There are two ways of supporting such instructions: one for Solidity and one for Yul. @@ -38,13 +38,13 @@ optimizing them out and is not emitting compilation errors. To see the list of available instructions, visit this page: -[zkSync Era Extension Simulation (call)](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/call.md) +[ZKsync Era Extension Simulation (call)](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/call.md) ### The Yul Mode -The non-call zkSync-specific instructions are only available in the Yul mode of **zksolc**. +The non-call ZKsync-specific instructions are only available in the Yul mode of **zksolc**. To have better compatibility, they are implemented as `verbatim` instructions with some predefined keys. To see the list of available instructions, visit this page: -[zkSync Era Extension Simulation (verbatim)](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/verbatim.md) +[ZKsync Era Extension Simulation (verbatim)](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions/extensions/verbatim.md) diff --git a/docs/specs/zk_evm/vm_specification/compiler/instructions/yul.md b/docs/specs/zk_evm/vm_specification/compiler/instructions/yul.md index 4841eee78521..55ae98166af8 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/instructions/yul.md +++ b/docs/specs/zk_evm/vm_specification/compiler/instructions/yul.md @@ -41,7 +41,7 @@ destination. For more information, see Writes immutables to the auxiliary heap. For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). LLVM IR codegen references: @@ -54,7 +54,7 @@ Reads immutables from the [ImmutableSimulator](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/system_contracts.md#simulator-of-immutables). For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/evm-instructions#setimmutable-loadimmutable). LLVM IR codegen references: @@ -71,7 +71,7 @@ compiler will return the list of deployable libraries not provided with `--libra like Hardhat to automatically deploy libraries. For more information, see the -[zkSync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/libraries). +[ZKsync Era documentation](https://docs.zksync.io/build/developer-reference/ethereum-differences/libraries). [The LLVM IR generator code](https://github.com/matter-labs/era-compiler-solidity/blob/main/src/yul/parser/statement/expression/function_call/mod.rs#L956). diff --git a/docs/specs/zk_evm/vm_specification/compiler/overview.md b/docs/specs/zk_evm/vm_specification/compiler/overview.md index d55b86756162..0af322e0b489 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/overview.md +++ b/docs/specs/zk_evm/vm_specification/compiler/overview.md @@ -8,7 +8,7 @@ | solc | The original Solidity compiler, developed by the Ethereum community. Called by zksolc as a subprocess to get the IRs of the source code of the project. | | LLVM | The compiler framework, used for optimizations and assembly generation. | | EraVM assembler/linker | The tool written in Rust. Translates the assembly emitted by LLVM to the target bytecode. | -| Virtual machine | The zkSync Era virtual machine called EraVM with a custom instruction set. | +| Virtual machine | The ZKsync Era virtual machine called EraVM with a custom instruction set. | | Intermediate representation (IR) | The data structure or code used internally by the compiler to represent source code. | | Yul | One of the Solidity IRs. Is a superset of the assembly available in Solidity. Used by default for contracts written in Solidity ≥0.8. | | EVMLA | One of the Solidity IRs called EVM legacy assembly. Is a predecessor of Yul, but must closer to the pure EVM bytecode. Used by default for contracts written in Solidity <0.8. | @@ -17,11 +17,11 @@ | EraVM bytecode | The smart contract bytecode, executed by EraVM. | | Stack | The segment of the non-persistent contract memory. Consists of two parts: 1. The global data, accessible from anywhere. Available to the compiler, not available to the user code. 2. The function-local stack frame without the depth limit like in EVM. | | Heap | The segment of the non-persistent contract memory. All the data is globally accessible by both the compiler and user code. The allocation is handled by the solc’s Yul/EVMLA allocator only. | -| Auxiliary heap | The segment of the non-persistent contract memory, introduced to avoid conflicts with the solc’s allocator. All the data is globally accessible by the compiler only. The allocation is handled by the zksolc’s compiler only. All contract calls specific to zkSync, including the system contracts, are made via the auxiliary heap. It is also used to return data (e.g. the array of immutables) from the constructor. | +| Auxiliary heap | The segment of the non-persistent contract memory, introduced to avoid conflicts with the solc’s allocator. All the data is globally accessible by the compiler only. The allocation is handled by the zksolc’s compiler only. All contract calls specific to ZKsync, including the system contracts, are made via the auxiliary heap. It is also used to return data (e.g. the array of immutables) from the constructor. | | Calldata | The segment of the non-persistent contract memory. The heap or auxiliary heap of the parent/caller contract. | | Return data | The segment of the non-persistent contract memory. The heap or auxiliary heap of the child/callee contract. | | Contract storage | The persistent contract memory. No relevant differences from that of EVM. | -| System contracts | The special set of zkSync kernel contracts written in Solidity by Matter Labs. | +| System contracts | The special set of ZKsync kernel contracts written in Solidity by Matter Labs. | | Contract context | The special storage of VM that keeps data like the current address, the caller’s address, etc. | ## Concepts @@ -95,7 +95,7 @@ the sake of assisting the upcoming audit. | calldataload | CALLDATALOAD | calldata | Stack: 1 input. Calldata: read 32 bytes. Stack: 1 output | - | 0 in deploy code. | | calldatacopy | CALLDATACOPY | calldata, heap | Stack: 3 inputs. Calldata: read N bytes. Heap: write N bytes | - | Generated by solc in the runtime code only. Copies 0 in deploy code. | | calldatasize | CALLDATASIZE | calldata | Stack: 1 output | - | 0 in deploy code. | -| codecopy | CODECOPY | calldata | Stack: 3 inputs. Calldata: read N bytes. Heap: write N bytes | - | Generated by solc in the deploy code only, but is treated as CALLDATACOPY, since the constructor arguments are calldata in zkSync 2.0. Compile time error in Yul runtime code. Copies 0 in EVMLA runtime code. | +| codecopy | CODECOPY | calldata | Stack: 3 inputs. Calldata: read N bytes. Heap: write N bytes | - | Generated by solc in the deploy code only, but is treated as CALLDATACOPY, since the constructor arguments are calldata in ZKsync 2.0. Compile time error in Yul runtime code. Copies 0 in EVMLA runtime code. | | codesize | CODESIZE | calldata | Stack: 1 output | - | - | | returndatacopy | RETURNDATACOPY | return data, heap | Stack: 3 inputs. Return data: read N bytes. Heap: write N bytes | - | - | | returndatasize | RETURNDATASIZE | return data | Stack: 1 output | - | - | @@ -137,6 +137,6 @@ the sake of assisting the upcoming audit. | pc | PC | unsupported | - | - | Compile time error | | selfdestruct | SELFDESTRUCT | unsupported | - | - | Compile time error | -For more information on how zkSync Era achieves EVM-equivalence, see the +For more information on how ZKsync Era achieves EVM-equivalence, see the [Instructions](https://github.com/code-423n4/2023-10-zksync/blob/main/docs/VM%20Section/How%20compiler%20works/instructions) section. diff --git a/docs/specs/zk_evm/vm_specification/compiler/system_contracts.md b/docs/specs/zk_evm/vm_specification/compiler/system_contracts.md index 0a68d0c4f299..3e328d663693 100644 --- a/docs/specs/zk_evm/vm_specification/compiler/system_contracts.md +++ b/docs/specs/zk_evm/vm_specification/compiler/system_contracts.md @@ -7,7 +7,7 @@ special handling, see ## Types of System Contracts -There are several types of System Contracts from the perspective of how they are handled by the zkSync Era compilers: +There are several types of System Contracts from the perspective of how they are handled by the ZKsync Era compilers: 1. [Environmental data storage](#environmental-data-storage). 2. [KECCAK256 hash function](#keccak256-hash-function). @@ -46,14 +46,14 @@ For reference, see Handling of this function is similar to [Environmental Data Storage](#environmental-data-storage) with one difference: Since EVM also uses heap to store the calldata for `KECCAK256`, the required memory chunk is allocated by the IR -generator, and zkSync Era compiler does not need to use [the auxiliary heap](#auxiliary-heap). +generator, and ZKsync Era compiler does not need to use [the auxiliary heap](#auxiliary-heap). For reference, see [the LLVM IR codegen source code](https://github.com/matter-labs/era-compiler-llvm-context/blob/main/src/eravm/context/function/llvm_runtime.rs). ### Contract Deployer -See [handling CREATE][docs-create] and [dependency code substitution instructions][docs-data] on zkSync Era +See [handling CREATE][docs-create] and [dependency code substitution instructions][docs-data] on ZKsync Era documentation. For reference, see LLVM IR codegen for @@ -85,7 +85,7 @@ For reference, see ### Simulator of Immutables -See [handling immutables][docs-immutable] on zkSync Era documentation. +See [handling immutables][docs-immutable] on ZKsync Era documentation. For reference, see LLVM IR codegen for [instructions for immutables](https://github.com/matter-labs/era-compiler-llvm-context/blob/main/src/eravm/evm/immutable.rs) diff --git a/etc/contracts-test-data/README.md b/etc/contracts-test-data/README.md index d08f934e8456..532703ad210f 100644 --- a/etc/contracts-test-data/README.md +++ b/etc/contracts-test-data/README.md @@ -1,4 +1,4 @@ # Contracts test data This folder contains data for contracts that are being used for testing to check the correctness of the smart contract -flow in zkSync. +flow in ZKsync. diff --git a/etc/contracts-test-data/contracts/custom-account/SystemContractsCaller.sol b/etc/contracts-test-data/contracts/custom-account/SystemContractsCaller.sol index e4d241116a1e..3ec2b81a1074 100644 --- a/etc/contracts-test-data/contracts/custom-account/SystemContractsCaller.sol +++ b/etc/contracts-test-data/contracts/custom-account/SystemContractsCaller.sol @@ -6,7 +6,7 @@ import {MSG_VALUE_SYSTEM_CONTRACT, MSG_VALUE_SIMULATOR_IS_SYSTEM_BIT} from "./Co import "./Utils.sol"; // Addresses used for the compiler to be replaced with the -// zkSync-specific opcodes during the compilation. +// ZKsync-specific opcodes during the compilation. // IMPORTANT: these are just compile-time constants and are used // only if used in-place by Yul optimizer. address constant TO_L1_CALL_ADDRESS = address((1 << 16) - 1); diff --git a/etc/contracts-test-data/contracts/custom-account/TransactionHelper.sol b/etc/contracts-test-data/contracts/custom-account/TransactionHelper.sol index 7fc883ed882c..82747b88d358 100644 --- a/etc/contracts-test-data/contracts/custom-account/TransactionHelper.sol +++ b/etc/contracts-test-data/contracts/custom-account/TransactionHelper.sol @@ -10,7 +10,7 @@ import "./interfaces/IContractDeployer.sol"; import {BASE_TOKEN_SYSTEM_CONTRACT, BOOTLOADER_FORMAL_ADDRESS} from "./Constants.sol"; import "./RLPEncoder.sol"; -/// @dev The type id of zkSync's EIP-712-signed transaction. +/// @dev The type id of ZKsync's EIP-712-signed transaction. uint8 constant EIP_712_TX_TYPE = 0x71; /// @dev The type id of legacy transactions. @@ -20,7 +20,7 @@ uint8 constant EIP_2930_TX_TYPE = 0x01; /// @dev The type id of EIP1559 transactions. uint8 constant EIP_1559_TX_TYPE = 0x02; -/// @notice Structure used to represent zkSync transaction. +/// @notice Structure used to represent ZKsync transaction. struct Transaction { // The type of the transaction. uint256 txType; @@ -118,7 +118,7 @@ library TransactionHelper { } } - /// @notice Encode hash of the zkSync native transaction type. + /// @notice Encode hash of the ZKsync native transaction type. /// @return keccak256 hash of the EIP-712 encoded representation of transaction function _encodeHashEIP712Transaction(Transaction calldata _transaction) private @@ -251,7 +251,7 @@ library TransactionHelper { // Hash of EIP2930 transactions is encoded the following way: // H(0x01 || RLP(chain_id, nonce, gas_price, gas_limit, destination, amount, data, access_list)) // - // Note, that on zkSync access lists are not supported and should always be empty. + // Note, that on ZKsync access lists are not supported and should always be empty. // Encode all fixed-length params to avoid "stack too deep error" bytes memory encodedFixedLengthParams; @@ -290,7 +290,7 @@ library TransactionHelper { // Otherwise the length is not encoded at all. } - // On zkSync, access lists are always zero length (at least for now). + // On ZKsync, access lists are always zero length (at least for now). bytes memory encodedAccessListLength = RLPEncoder.encodeListLen(0); bytes memory encodedListLength; @@ -327,7 +327,7 @@ library TransactionHelper { // Hash of EIP1559 transactions is encoded the following way: // H(0x02 || RLP(chain_id, nonce, max_priority_fee_per_gas, max_fee_per_gas, gas_limit, destination, amount, data, access_list)) // - // Note, that on zkSync access lists are not supported and should always be empty. + // Note, that on ZKsync access lists are not supported and should always be empty. // Encode all fixed-length params to avoid "stack too deep error" bytes memory encodedFixedLengthParams; @@ -368,7 +368,7 @@ library TransactionHelper { // Otherwise the length is not encoded at all. } - // On zkSync, access lists are always zero length (at least for now). + // On ZKsync, access lists are always zero length (at least for now). bytes memory encodedAccessListLength = RLPEncoder.encodeListLen(0); bytes memory encodedListLength; diff --git a/etc/contracts-test-data/contracts/custom-account/Utils.sol b/etc/contracts-test-data/contracts/custom-account/Utils.sol index da3d4eb60878..e562948942d7 100644 --- a/etc/contracts-test-data/contracts/custom-account/Utils.sol +++ b/etc/contracts-test-data/contracts/custom-account/Utils.sol @@ -3,7 +3,7 @@ pragma solidity >=0.8.0; /** * @author Matter Labs - * @dev Common utilities used in zkSync system contracts + * @dev Common utilities used in ZKsync system contracts */ library Utils { function safeCastToU128(uint256 _x) internal pure returns (uint128) { diff --git a/etc/env/base/README.md b/etc/env/base/README.md index 8bf4ceb48cdb..c583685d953c 100644 --- a/etc/env/base/README.md +++ b/etc/env/base/README.md @@ -1,6 +1,6 @@ -# Base configuration for zkSync stack +# Base configuration for ZKsync stack -This folder contains the template for generating the configuration for zkSync applications. Configs in this folder are +This folder contains the template for generating the configuration for ZKsync applications. Configs in this folder are assigned default values suitable for development. Since all the applications expect configuration to be set via the environment variables, these configs are compiled into diff --git a/etc/env/base/chain.toml b/etc/env/base/chain.toml index 8e0c37b76935..88a4c71bbb9b 100644 --- a/etc/env/base/chain.toml +++ b/etc/env/base/chain.toml @@ -1,13 +1,13 @@ -# zkSync chain parameters +# ZKsync chain parameters [chain.eth] # Name of the used Ethereum network network = "localhost" -# Name of current zkSync network +# Name of current ZKsync network # Used for Sentry environment zksync_network = "localhost" -# ID of current zkSync network treated as ETH network ID. -# Used to distinguish zkSync from other Web3-capable networks. +# ID of current ZKsync network treated as ETH network ID. +# Used to distinguish ZKsync from other Web3-capable networks. zksync_network_id = 270 [chain.state_keeper] @@ -78,10 +78,10 @@ max_gas_per_batch = 200000000 max_pubdata_per_batch = 100000 # The version of the fee model to use. -# - `V1`, the first model that was used in zkSync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. +# - `V1`, the first model that was used in ZKsync Era. In this fee model, the pubdata price must be pegged to the L1 gas price. # Also, the fair L2 gas price is expected to only include the proving/computation price for the operator and not the costs that come from # processing the batch on L1. -# - `V2`, the second model that was used in zkSync Era. There the pubdata price might be independent from the L1 gas price. Also, +# - `V2`, the second model that was used in ZKsync Era. There the pubdata price might be independent from the L1 gas price. Also, # The fair L2 gas price is expected to both the proving/computation price for the operator and the costs that come from # processing the batch on L1. fee_model_version = "V1" diff --git a/etc/env/base/contracts.toml b/etc/env/base/contracts.toml index b88a3e179eae..491bd19ea4bf 100644 --- a/etc/env/base/contracts.toml +++ b/etc/env/base/contracts.toml @@ -1,4 +1,4 @@ -# Addresses of the deployed zkSync contracts. +# Addresses of the deployed ZKsync contracts. # Values of this file are updated automatically by the contract deploy script. [contracts] diff --git a/etc/env/base/private.toml b/etc/env/base/private.toml index 1d6f8dabf82b..e6367e013519 100644 --- a/etc/env/base/private.toml +++ b/etc/env/base/private.toml @@ -9,7 +9,7 @@ test_database_prover_url = "postgres://postgres:notsecurepassword@localhost:5433 [eth_sender.sender] # Set in env file for development, production, staging and testnet. operator_private_key = "0x27593fea79697e947890ecbecce7901b0008345e5d7259710d0dd5e500d040be" -# Address to be used for zkSync account managing the interaction with a contract on Ethereum. +# Address to be used for ZKsync account managing the interaction with a contract on Ethereum. # Derived from the `OPERATOR_PRIVATE_KEY`. operator_commit_eth_addr = "0xde03a0B5963f75f1C8485B355fF6D30f3093BDE7" diff --git a/etc/test_config/README.md b/etc/test_config/README.md index ac7ecffd4ec7..3ec67f196736 100644 --- a/etc/test_config/README.md +++ b/etc/test_config/README.md @@ -1,6 +1,6 @@ -# Test data for zkSync +# Test data for ZKsync -This folder contains the data required for various zkSync tests. +This folder contains the data required for various ZKsync tests. Directory contains three subfolders: diff --git a/flake.nix b/flake.nix index 018aebb15da9..0287d4cf09d1 100644 --- a/flake.nix +++ b/flake.nix @@ -20,7 +20,7 @@ # ################################################################################################### { - description = "zkSync-era"; + description = "ZKsync-era"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; flake-utils.url = "github:numtide/flake-utils"; diff --git a/infrastructure/local-setup-preparation/README.md b/infrastructure/local-setup-preparation/README.md index fe59f930bf4c..6f3c97961434 100644 --- a/infrastructure/local-setup-preparation/README.md +++ b/infrastructure/local-setup-preparation/README.md @@ -1,6 +1,6 @@ # Scripts for local setup preparation -This project contains scripts that should be executed when preparing the zkSync local setup used by outside developers, +This project contains scripts that should be executed when preparing the ZKsync local setup used by outside developers, e.g. deposit ETH to some of the test accounts. With the server running (`zk server`), execute the following from `$ZKSYNC_HOME` to fund the L2 wallets diff --git a/infrastructure/local-setup-preparation/src/index.ts b/infrastructure/local-setup-preparation/src/index.ts index 805d13aadcdb..9d8b7efea66d 100644 --- a/infrastructure/local-setup-preparation/src/index.ts +++ b/infrastructure/local-setup-preparation/src/index.ts @@ -16,7 +16,7 @@ async function depositWithRichAccounts() { const handles: Promise[] = []; if (!process.env.CONTRACTS_BRIDGEHUB_PROXY_ADDR) { - throw new Error('zkSync L1 Main contract address was not found'); + throw new Error('ZKsync L1 Main contract address was not found'); } // During the preparation for the local node, the L2 server is not available, so diff --git a/infrastructure/protocol-upgrade/src/index.ts b/infrastructure/protocol-upgrade/src/index.ts index c94d280495f2..d78726437852 100644 --- a/infrastructure/protocol-upgrade/src/index.ts +++ b/infrastructure/protocol-upgrade/src/index.ts @@ -15,7 +15,7 @@ async function main() { const ZKSYNC_HOME = process.env.ZKSYNC_HOME; if (!ZKSYNC_HOME) { - throw new Error('Please set $ZKSYNC_HOME to the root of zkSync repo!'); + throw new Error('Please set $ZKSYNC_HOME to the root of ZKsync repo!'); } else { process.chdir(ZKSYNC_HOME); } diff --git a/infrastructure/zk/src/index.ts b/infrastructure/zk/src/index.ts index 1fd05252a591..0c11c110c6e3 100644 --- a/infrastructure/zk/src/index.ts +++ b/infrastructure/zk/src/index.ts @@ -60,7 +60,7 @@ async function main() { const ZKSYNC_HOME = process.env.ZKSYNC_HOME; if (!ZKSYNC_HOME) { - throw new Error('Please set $ZKSYNC_HOME to the root of zkSync repo!'); + throw new Error('Please set $ZKSYNC_HOME to the root of ZKsync repo!'); } else { process.chdir(ZKSYNC_HOME); } diff --git a/prover/prover_cli/README.md b/prover/prover_cli/README.md index 74f291c8d573..053744914b97 100644 --- a/prover/prover_cli/README.md +++ b/prover/prover_cli/README.md @@ -1,6 +1,6 @@ # Prover CLI -CLI tool for performing maintenance of a zkSync Prover +CLI tool for performing maintenance of a ZKsync Prover ## Installation diff --git a/zk_toolbox/Cargo.toml b/zk_toolbox/Cargo.toml index 6f9c288438ed..15e1ddc4cdcb 100644 --- a/zk_toolbox/Cargo.toml +++ b/zk_toolbox/Cargo.toml @@ -17,7 +17,7 @@ authors = ["The Matter Labs Team "] exclude = ["./github"] repository = "https://github.com/matter-labs/zk_toolbox/" description = "ZK Toolbox is a set of tools for working with zk stack." -keywords = ["zk", "cryptography", "blockchain", "ZKStack", "zkSync"] +keywords = ["zk", "cryptography", "blockchain", "ZKStack", "ZKsync"] [workspace.dependencies] diff --git a/zk_toolbox/crates/common/src/term/logger.rs b/zk_toolbox/crates/common/src/term/logger.rs index b505123114be..33a88bd961e2 100644 --- a/zk_toolbox/crates/common/src/term/logger.rs +++ b/zk_toolbox/crates/common/src/term/logger.rs @@ -14,7 +14,7 @@ fn term_write(msg: impl Display) { } pub fn intro() { - cliclak_intro(style(" zkSync toolbox ").on_cyan().black()).unwrap(); + cliclak_intro(style(" ZKsync toolbox ").on_cyan().black()).unwrap(); } pub fn outro(msg: impl Display) { diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 90645ff19acf..9141d044af94 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -18,7 +18,7 @@ pub(crate) const INITIAL_DEPLOYMENT_FILE: &str = "initial_deployments.yaml"; pub(crate) const ERC20_DEPLOYMENT_FILE: &str = "erc20_deployments.yaml"; /// Name of the contracts file pub(crate) const CONTRACTS_FILE: &str = "contracts.yaml"; -/// Main repository for the zkSync project +/// Main repository for the ZKsync project pub const ZKSYNC_ERA_GIT_REPO: &str = "https://github.com/matter-labs/zksync-era"; /// Name of the docker-compose file inside zksync repository pub const DOCKER_COMPOSE_FILE: &str = "docker-compose.yml"; From c8282173494b6a8ae8f4c4c96936f2f6564d55fe Mon Sep 17 00:00:00 2001 From: pompon0 Date: Wed, 19 Jun 2024 14:48:17 +0200 Subject: [PATCH 22/43] chore: bumped curve25519 (#2274) https://rustsec.org/advisories/RUSTSEC-2024-0344 --- Cargo.lock | 11 ++--------- prover/Cargo.lock | 11 ++--------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be0ffd1566b1..f7549eac1549 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1561,16 +1561,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -4459,12 +4458,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "platforms" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" - [[package]] name = "plotters" version = "0.3.5" diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 7b30b67c2650..f65d90ff40c2 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -1668,16 +1668,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -4649,12 +4648,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "platforms" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" - [[package]] name = "poly1305" version = "0.8.0" From 440f2a7ae0def22bab65c4bb5c531b3234841b76 Mon Sep 17 00:00:00 2001 From: Joaquin Carletti <56092489+ColoCarletti@users.noreply.github.com> Date: Wed, 19 Jun 2024 09:59:57 -0300 Subject: [PATCH 23/43] fix(prover_cli): Fix Minor Bugs in Prover CLI (#2264) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes bugs in the Prover CLI: - The status command was not displaying correctly for more than one batch. - The witness job status was incorrectly set to "in progress" when some batches were in the queue and others were waiting for proofs. - The config command failed if there was no configuration file. Now, it creates one if it’s not found. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- prover/prover_cli/src/commands/status/batch.rs | 4 ++-- prover/prover_cli/src/commands/status/utils.rs | 10 +++++----- prover/prover_cli/src/config/mod.rs | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/prover/prover_cli/src/commands/status/batch.rs b/prover/prover_cli/src/commands/status/batch.rs index 0279fd658f60..dc23bf046683 100644 --- a/prover/prover_cli/src/commands/status/batch.rs +++ b/prover/prover_cli/src/commands/status/batch.rs @@ -39,7 +39,7 @@ pub(crate) async fn run(args: Args, config: ProverCLIConfig) -> anyhow::Result<( if let Status::Custom(msg) = batch_data.compressor.witness_generator_jobs_status() { if msg.contains("Sent to server") { println!("> Proof sent to server ✅"); - return Ok(()); + continue; } } @@ -48,7 +48,7 @@ pub(crate) async fn run(args: Args, config: ProverCLIConfig) -> anyhow::Result<( .witness_generator_jobs_status(); if matches!(basic_witness_generator_status, Status::JobsNotFound) { println!("> No batch found. 🚫"); - return Ok(()); + continue; } if !args.verbose { diff --git a/prover/prover_cli/src/commands/status/utils.rs b/prover/prover_cli/src/commands/status/utils.rs index 59c5553b530b..31726e749209 100644 --- a/prover/prover_cli/src/commands/status/utils.rs +++ b/prover/prover_cli/src/commands/status/utils.rs @@ -75,16 +75,16 @@ impl From> for Status { fn from(status_vector: Vec) -> Self { if status_vector.is_empty() { Status::JobsNotFound - } else if status_vector - .iter() - .all(|job| matches!(job, WitnessJobStatus::Queued)) - { - Status::Queued } else if status_vector .iter() .all(|job| matches!(job, WitnessJobStatus::WaitingForProofs)) { Status::WaitingForProofs + } else if status_vector.iter().all(|job| { + matches!(job, WitnessJobStatus::Queued) + || matches!(job, WitnessJobStatus::WaitingForProofs) + }) { + Status::Queued } else if status_vector .iter() .all(|job| matches!(job, WitnessJobStatus::Successful(_))) diff --git a/prover/prover_cli/src/config/mod.rs b/prover/prover_cli/src/config/mod.rs index 93af17317c5d..3d99f2be3b2c 100644 --- a/prover/prover_cli/src/config/mod.rs +++ b/prover/prover_cli/src/config/mod.rs @@ -26,7 +26,7 @@ pub fn update_envfile( let prefix = format!("{}=", key.as_ref()); let kv = format!("{}={}", key.as_ref(), value.as_ref()); let swapfile = path.as_ref().with_extension(".swp"); - let mut out = std::io::BufWriter::new(std::fs::File::create_new(&swapfile)?); + let mut out = std::io::BufWriter::new(std::fs::File::create(&swapfile)?); let mut found = false; std::fs::read_to_string(path) From 15bb71ec3ee66796e62cb7e61dec6e496e1f2774 Mon Sep 17 00:00:00 2001 From: perekopskiy <53865202+perekopskiy@users.noreply.github.com> Date: Wed, 19 Jun 2024 16:01:13 +0300 Subject: [PATCH 24/43] fix(vm): fix insertion to `decommitted_code_hashes` (#2275) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- .../versions/vm_latest/old_vm/oracles/decommitter.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs b/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs index 7c7dc6995d14..f5cd38779217 100644 --- a/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs +++ b/core/lib/multivm/src/versions/vm_latest/old_vm/oracles/decommitter.rs @@ -180,8 +180,15 @@ impl DecommittmentProcess Ok(partial_query) } else { partial_query.is_fresh = true; - self.decommitted_code_hashes - .insert(stored_hash, None, partial_query.timestamp); + if self + .decommitted_code_hashes + .inner() + .get(&stored_hash) + .is_none() + { + self.decommitted_code_hashes + .insert(stored_hash, None, partial_query.timestamp); + } Ok(partial_query) } From 2c0a00add179cc4ed521bbb9d616b8828f0ad3c1 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 19 Jun 2024 15:09:01 +0200 Subject: [PATCH 25/43] feat(tee_verifier_input_producer): use `FactoryDepsDal::get_factory_deps() (#2271) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Use `FactoryDepsDal::get_factory_deps()` in the tee_verifier_input_producer crate. ## Why ❔ This optimizes getting the system contracts and gets rid of a couple of workarounds. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. Signed-off-by: Harald Hoyer --- Cargo.lock | 2 - .../tee_verifier_input_producer/Cargo.toml | 2 - .../tee_verifier_input_producer/src/lib.rs | 70 +++++++------------ 3 files changed, 24 insertions(+), 50 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f7549eac1549..1be8739e881d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9364,7 +9364,6 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "multivm", "tokio", "tracing", "vise", @@ -9373,7 +9372,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_interface", "zksync_queued_job_processor", - "zksync_state", "zksync_tee_verifier", "zksync_types", "zksync_utils", diff --git a/core/node/tee_verifier_input_producer/Cargo.toml b/core/node/tee_verifier_input_producer/Cargo.toml index 49856f5c7022..208e7e35760c 100644 --- a/core/node/tee_verifier_input_producer/Cargo.toml +++ b/core/node/tee_verifier_input_producer/Cargo.toml @@ -14,11 +14,9 @@ zksync_dal.workspace = true zksync_object_store.workspace = true zksync_prover_interface.workspace = true zksync_queued_job_processor.workspace = true -zksync_state.workspace = true zksync_tee_verifier.workspace = true zksync_types.workspace = true zksync_utils.workspace = true -multivm.workspace = true vm_utils.workspace = true vise.workspace = true diff --git a/core/node/tee_verifier_input_producer/src/lib.rs b/core/node/tee_verifier_input_producer/src/lib.rs index efa3c9e00b11..9104b62fa5e5 100644 --- a/core/node/tee_verifier_input_producer/src/lib.rs +++ b/core/node/tee_verifier_input_producer/src/lib.rs @@ -11,16 +11,14 @@ use std::{sync::Arc, time::Instant}; use anyhow::Context; use async_trait::async_trait; -use multivm::zk_evm_latest::ethereum_types::H256; -use tokio::{runtime::Handle, task::JoinHandle}; +use tokio::task::JoinHandle; use vm_utils::storage::L1BatchParamsProvider; use zksync_dal::{tee_verifier_input_producer_dal::JOB_MAX_ATTEMPT, ConnectionPool, Core, CoreDal}; use zksync_object_store::ObjectStore; use zksync_prover_interface::inputs::PrepareBasicCircuitsJob; use zksync_queued_job_processor::JobProcessor; -use zksync_state::{PostgresStorage, ReadStorage}; use zksync_tee_verifier::TeeVerifierInput; -use zksync_types::{block::L1BatchHeader, L1BatchNumber, L2BlockNumber, L2ChainId}; +use zksync_types::{L1BatchNumber, L2ChainId}; use zksync_utils::u256_to_h256; use self::metrics::METRICS; @@ -49,7 +47,6 @@ impl TeeVerifierInputProducer { } async fn process_job_impl( - rt_handle: Handle, l1_batch_number: L1BatchNumber, started_at: Instant, connection_pool: ConnectionPool, @@ -71,8 +68,6 @@ impl TeeVerifierInputProducer { .get_l2_blocks_to_execute_for_l1_batch(l1_batch_number) .await?; - let last_batch_miniblock_number = l2_blocks_execution_data.first().unwrap().number - 1; - let l1_batch_header = connection .blocks_dal() .get_l1_batch_header(l1_batch_number) @@ -107,19 +102,29 @@ impl TeeVerifierInputProducer { .await .context("expected miniblock to be executed and sealed")?; - // need a new connection in the next block - drop(connection); + let used_contract_hashes = l1_batch_header + .used_contract_hashes + .into_iter() + .map(u256_to_h256) + .collect(); + + // `get_factory_deps()` returns the bytecode in chunks of `Vec<[u8; 32]>`, + // but `fn store_factory_dep(&mut self, hash: H256, bytecode: Vec)` in `InMemoryStorage` wants flat byte vecs. + pub fn into_flattened(data: Vec<[T; N]>) -> Vec { + let mut new = Vec::new(); + for slice in data.iter() { + new.extend_from_slice(slice); + } + new + } - // `PostgresStorage` needs a blocking context - let used_contracts = rt_handle - .spawn_blocking(move || { - Self::get_used_contracts( - last_batch_miniblock_number, - l1_batch_header, - connection_pool, - ) - }) - .await??; + let used_contracts = connection + .factory_deps_dal() + .get_factory_deps(&used_contract_hashes) + .await + .into_iter() + .map(|(hash, bytes)| (u256_to_h256(hash), into_flattened(bytes))) + .collect(); tracing::info!("Started execution of l1_batch: {l1_batch_number:?}"); @@ -146,31 +151,6 @@ impl TeeVerifierInputProducer { Ok(tee_verifier_input) } - - fn get_used_contracts( - last_batch_miniblock_number: L2BlockNumber, - l1_batch_header: L1BatchHeader, - connection_pool: ConnectionPool, - ) -> anyhow::Result)>> { - let rt_handle = Handle::current(); - - let connection = rt_handle - .block_on(connection_pool.connection()) - .context("failed to get connection for TeeVerifierInputProducer")?; - - let mut pg_storage = - PostgresStorage::new(rt_handle, connection, last_batch_miniblock_number, true); - - Ok(l1_batch_header - .used_contract_hashes - .into_iter() - .filter_map(|hash| { - pg_storage - .load_factory_dep(u256_to_h256(hash)) - .map(|bytes| (u256_to_h256(hash), bytes)) - }) - .collect()) - } } #[async_trait] @@ -217,9 +197,7 @@ impl JobProcessor for TeeVerifierInputProducer { let connection_pool = self.connection_pool.clone(); let object_store = self.object_store.clone(); tokio::task::spawn(async move { - let rt_handle = Handle::current(); Self::process_job_impl( - rt_handle, job, started_at, connection_pool.clone(), From 05c6f357eee591262e3ddd870fcde0fe50ce05cc Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Wed, 19 Jun 2024 15:50:34 +0100 Subject: [PATCH 26/43] fix: prover Cargo.lock (#2280) --- prover/Cargo.lock | 2 -- 1 file changed, 2 deletions(-) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index f65d90ff40c2..096e3998d0a4 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -9508,7 +9508,6 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "multivm", "tokio", "tracing", "vise", @@ -9517,7 +9516,6 @@ dependencies = [ "zksync_object_store", "zksync_prover_interface", "zksync_queued_job_processor", - "zksync_state", "zksync_tee_verifier", "zksync_types", "zksync_utils", From 3bf8966dba2d76a1e8fdd5f7126a915cc9b5e3cb Mon Sep 17 00:00:00 2001 From: Joonatan Saarhelo Date: Wed, 19 Jun 2024 16:38:13 +0100 Subject: [PATCH 27/43] chore: use simple StorageLog struct instead of LogQuery when possible (#2252) There are many fields in LogQuery on the core side (it also exists in zk_evm) that are unused and often just set to some garbage value but that is very confusing. This PR changes the uses of LogQuery in the VmInterface to StorageLog, which only has the necessary fields. --------- Co-authored-by: perekopskiy --- core/bin/snapshots_creator/src/tests.rs | 2 +- ...208a30b0eead764527ff957ea6e86a34eec6.json} | 10 +- ...5a8869da8f10738ba77e3d8e048057b0e7b12.json | 36 +++ ...f0a9676e26f422506545ccc90b7e8a36c8d47.json | 35 --- ...ake_storage_logs_tx_hash_nullable.down.sql | 1 + ..._make_storage_logs_tx_hash_nullable.up.sql | 1 + core/lib/dal/src/contract_verification_dal.rs | 37 +-- core/lib/dal/src/models/storage_log.rs | 1 - core/lib/dal/src/pruning_dal/tests.rs | 2 +- core/lib/dal/src/snapshots_creator_dal.rs | 8 +- core/lib/dal/src/storage_logs_dal.rs | 45 ++-- core/lib/dal/src/storage_logs_dedup_dal.rs | 11 +- core/lib/dal/src/tokens_dal.rs | 11 +- core/lib/multivm/src/glue/types/vm/mod.rs | 1 + .../multivm/src/glue/types/vm/storage_log.rs | 29 +++ .../src/glue/types/vm/vm_block_result.rs | 58 +++-- .../types/outputs/execution_result.rs | 4 +- .../types/outputs/execution_state.rs | 4 +- .../src/versions/vm_1_3_2/oracles/storage.rs | 12 +- .../multivm/src/versions/vm_1_3_2/utils.rs | 4 +- core/lib/multivm/src/versions/vm_1_3_2/vm.rs | 2 +- .../versions/vm_1_4_1/implementation/logs.rs | 5 +- .../src/versions/vm_1_4_1/oracles/storage.rs | 12 +- .../src/versions/vm_1_4_1/utils/logs.rs | 4 +- core/lib/multivm/src/versions/vm_1_4_1/vm.rs | 2 +- .../versions/vm_1_4_2/implementation/logs.rs | 5 +- .../src/versions/vm_1_4_2/oracles/storage.rs | 12 +- .../src/versions/vm_1_4_2/utils/logs.rs | 4 +- core/lib/multivm/src/versions/vm_1_4_2/vm.rs | 2 +- .../implementation/logs.rs | 5 +- .../vm_boojum_integration/oracles/storage.rs | 12 +- .../vm_boojum_integration/utils/logs.rs | 4 +- .../src/versions/vm_boojum_integration/vm.rs | 2 +- .../versions/vm_latest/implementation/logs.rs | 5 +- .../src/versions/vm_latest/oracles/storage.rs | 12 +- .../vm_latest/tests/l1_tx_execution.rs | 2 +- .../src/versions/vm_latest/tests/refunds.rs | 12 +- .../src/versions/vm_latest/utils/logs.rs | 4 +- core/lib/multivm/src/versions/vm_latest/vm.rs | 2 +- .../src/versions/vm_m5/oracles/storage.rs | 12 +- core/lib/multivm/src/versions/vm_m5/utils.rs | 4 +- core/lib/multivm/src/versions/vm_m5/vm.rs | 2 +- .../src/versions/vm_m6/oracles/storage.rs | 12 +- core/lib/multivm/src/versions/vm_m6/utils.rs | 4 +- core/lib/multivm/src/versions/vm_m6/vm.rs | 2 +- .../vm_refunds_enhancement/oracles/storage.rs | 12 +- .../vm_refunds_enhancement/utils/logs.rs | 4 +- .../src/versions/vm_refunds_enhancement/vm.rs | 2 +- .../old_vm/oracles/storage.rs | 12 +- .../versions/vm_virtual_blocks/utils/logs.rs | 4 +- .../src/versions/vm_virtual_blocks/vm.rs | 2 +- core/lib/state/src/in_memory.rs | 6 +- core/lib/state/src/test_utils.rs | 7 +- core/lib/tee_verifier/src/lib.rs | 29 +-- core/lib/types/src/storage/log.rs | 80 +++++-- .../types/src/storage_writes_deduplicator.rs | 223 ++++++++---------- core/node/api_server/src/tx_sender/proxy.rs | 4 +- core/node/api_server/src/tx_sender/tests.rs | 11 +- .../web3/backend_jsonrpsee/namespaces/zks.rs | 4 +- core/node/api_server/src/web3/tests/mod.rs | 6 +- core/node/api_server/src/web3/tests/vm.rs | 71 +++--- core/node/block_reverter/src/tests.rs | 5 +- core/node/commitment_generator/src/tests.rs | 2 +- core/node/genesis/src/utils.rs | 82 +++---- core/node/metadata_calculator/src/helpers.rs | 6 +- core/node/metadata_calculator/src/tests.rs | 2 +- .../src/batch_executor/tests/tester.rs | 8 +- core/node/state_keeper/src/io/persistence.rs | 72 +++--- .../io/seal_logic/l2_block_seal_subtasks.rs | 30 +-- .../state_keeper/src/io/seal_logic/mod.rs | 67 +----- core/node/state_keeper/src/io/tests/mod.rs | 10 +- core/node/state_keeper/src/mempool_actor.rs | 4 +- .../state_keeper/src/seal_criteria/mod.rs | 2 +- core/node/state_keeper/src/testonly/mod.rs | 8 +- core/node/state_keeper/src/tests/mod.rs | 52 ++-- .../src/updates/l1_batch_updates.rs | 2 +- .../src/updates/l2_block_updates.rs | 6 +- core/node/state_keeper/src/updates/mod.rs | 2 +- core/node/test_utils/src/lib.rs | 5 +- .../vm_runner/src/impls/protective_reads.rs | 17 +- core/node/vm_runner/src/tests/mod.rs | 6 +- 81 files changed, 607 insertions(+), 714 deletions(-) rename core/lib/dal/.sqlx/{query-c75cdc655cd843a474f857e80b30685582bb37ba816a5434ee546d86ef9a8d9e.json => query-21cfb584e3731852e96da1968503208a30b0eead764527ff957ea6e86a34eec6.json} (67%) create mode 100644 core/lib/dal/.sqlx/query-2cba440c2925631655a7f67486a5a8869da8f10738ba77e3d8e048057b0e7b12.json delete mode 100644 core/lib/dal/.sqlx/query-9955b9215096f781442153518c4f0a9676e26f422506545ccc90b7e8a36c8d47.json create mode 100644 core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.down.sql create mode 100644 core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.up.sql create mode 100644 core/lib/multivm/src/glue/types/vm/storage_log.rs diff --git a/core/bin/snapshots_creator/src/tests.rs b/core/bin/snapshots_creator/src/tests.rs index 59c0e853a621..4fd553d0348d 100644 --- a/core/bin/snapshots_creator/src/tests.rs +++ b/core/bin/snapshots_creator/src/tests.rs @@ -159,7 +159,7 @@ async fn create_l2_block( .await .unwrap(); conn.storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(H256::zero(), block_logs)]) + .insert_storage_logs(l2_block_number, &block_logs) .await .unwrap(); } diff --git a/core/lib/dal/.sqlx/query-c75cdc655cd843a474f857e80b30685582bb37ba816a5434ee546d86ef9a8d9e.json b/core/lib/dal/.sqlx/query-21cfb584e3731852e96da1968503208a30b0eead764527ff957ea6e86a34eec6.json similarity index 67% rename from core/lib/dal/.sqlx/query-c75cdc655cd843a474f857e80b30685582bb37ba816a5434ee546d86ef9a8d9e.json rename to core/lib/dal/.sqlx/query-21cfb584e3731852e96da1968503208a30b0eead764527ff957ea6e86a34eec6.json index 0cf33a5559f0..6d78d4ebd2f0 100644 --- a/core/lib/dal/.sqlx/query-c75cdc655cd843a474f857e80b30685582bb37ba816a5434ee546d86ef9a8d9e.json +++ b/core/lib/dal/.sqlx/query-21cfb584e3731852e96da1968503208a30b0eead764527ff957ea6e86a34eec6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT\n hashed_key,\n address,\n key,\n value,\n operation_number,\n tx_hash,\n miniblock_number\n FROM\n storage_logs\n ORDER BY\n miniblock_number,\n operation_number\n ", + "query": "\n SELECT\n hashed_key,\n address,\n key,\n value,\n operation_number,\n miniblock_number\n FROM\n storage_logs\n ORDER BY\n miniblock_number,\n operation_number\n ", "describe": { "columns": [ { @@ -30,11 +30,6 @@ }, { "ordinal": 5, - "name": "tx_hash", - "type_info": "Bytea" - }, - { - "ordinal": 6, "name": "miniblock_number", "type_info": "Int8" } @@ -48,9 +43,8 @@ false, false, false, - false, false ] }, - "hash": "c75cdc655cd843a474f857e80b30685582bb37ba816a5434ee546d86ef9a8d9e" + "hash": "21cfb584e3731852e96da1968503208a30b0eead764527ff957ea6e86a34eec6" } diff --git a/core/lib/dal/.sqlx/query-2cba440c2925631655a7f67486a5a8869da8f10738ba77e3d8e048057b0e7b12.json b/core/lib/dal/.sqlx/query-2cba440c2925631655a7f67486a5a8869da8f10738ba77e3d8e048057b0e7b12.json new file mode 100644 index 000000000000..b01a5b41649b --- /dev/null +++ b/core/lib/dal/.sqlx/query-2cba440c2925631655a7f67486a5a8869da8f10738ba77e3d8e048057b0e7b12.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n factory_deps.bytecode,\n transactions.data AS \"data?\",\n transactions.contract_address AS \"contract_address?\"\n FROM\n (\n SELECT\n miniblock_number,\n tx_hash,\n topic3\n FROM\n events\n WHERE\n address = $1\n AND topic1 = $2\n AND topic4 = $3\n LIMIT\n 1\n ) deploy_event\n JOIN factory_deps ON factory_deps.bytecode_hash = deploy_event.topic3\n LEFT JOIN transactions ON transactions.hash = deploy_event.tx_hash\n WHERE\n deploy_event.miniblock_number <= (\n SELECT\n MAX(number)\n FROM\n miniblocks\n )\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "bytecode", + "type_info": "Bytea" + }, + { + "ordinal": 1, + "name": "data?", + "type_info": "Jsonb" + }, + { + "ordinal": 2, + "name": "contract_address?", + "type_info": "Bytea" + } + ], + "parameters": { + "Left": [ + "Bytea", + "Bytea", + "Bytea" + ] + }, + "nullable": [ + false, + true, + true + ] + }, + "hash": "2cba440c2925631655a7f67486a5a8869da8f10738ba77e3d8e048057b0e7b12" +} diff --git a/core/lib/dal/.sqlx/query-9955b9215096f781442153518c4f0a9676e26f422506545ccc90b7e8a36c8d47.json b/core/lib/dal/.sqlx/query-9955b9215096f781442153518c4f0a9676e26f422506545ccc90b7e8a36c8d47.json deleted file mode 100644 index c05539164cee..000000000000 --- a/core/lib/dal/.sqlx/query-9955b9215096f781442153518c4f0a9676e26f422506545ccc90b7e8a36c8d47.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT\n factory_deps.bytecode,\n transactions.data AS \"data?\",\n transactions.contract_address AS \"contract_address?\"\n FROM\n (\n SELECT\n *\n FROM\n storage_logs\n WHERE\n storage_logs.hashed_key = $1\n ORDER BY\n miniblock_number DESC,\n operation_number DESC\n LIMIT\n 1\n ) storage_logs\n JOIN factory_deps ON factory_deps.bytecode_hash = storage_logs.value\n LEFT JOIN transactions ON transactions.hash = storage_logs.tx_hash\n WHERE\n storage_logs.value != $2\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "bytecode", - "type_info": "Bytea" - }, - { - "ordinal": 1, - "name": "data?", - "type_info": "Jsonb" - }, - { - "ordinal": 2, - "name": "contract_address?", - "type_info": "Bytea" - } - ], - "parameters": { - "Left": [ - "Bytea", - "Bytea" - ] - }, - "nullable": [ - false, - false, - true - ] - }, - "hash": "9955b9215096f781442153518c4f0a9676e26f422506545ccc90b7e8a36c8d47" -} diff --git a/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.down.sql b/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.down.sql new file mode 100644 index 000000000000..c1c97e67e442 --- /dev/null +++ b/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.down.sql @@ -0,0 +1 @@ +ALTER TABLE storage_logs ALTER COLUMN tx_hash SET NOT NULL; diff --git a/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.up.sql b/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.up.sql new file mode 100644 index 000000000000..d6c0e5d2b36b --- /dev/null +++ b/core/lib/dal/migrations/20240619060210_make_storage_logs_tx_hash_nullable.up.sql @@ -0,0 +1 @@ +ALTER TABLE storage_logs ALTER COLUMN tx_hash DROP NOT NULL; diff --git a/core/lib/dal/src/contract_verification_dal.rs b/core/lib/dal/src/contract_verification_dal.rs index 03c6c408f654..3045c84255ee 100644 --- a/core/lib/dal/src/contract_verification_dal.rs +++ b/core/lib/dal/src/contract_verification_dal.rs @@ -12,8 +12,10 @@ use zksync_types::{ DeployContractCalldata, VerificationIncomingRequest, VerificationInfo, VerificationRequest, VerificationRequestStatus, }, - get_code_key, Address, CONTRACT_DEPLOYER_ADDRESS, FAILED_CONTRACT_DEPLOYMENT_BYTECODE_HASH, + event::DEPLOY_EVENT_SIGNATURE, + Address, CONTRACT_DEPLOYER_ADDRESS, }; +use zksync_utils::address_to_h256; use crate::{models::storage_verification_request::StorageVerificationRequest, Core}; @@ -288,7 +290,7 @@ impl ContractVerificationDal<'_, '_> { &mut self, address: Address, ) -> anyhow::Result, DeployContractCalldata)>> { - let hashed_key = get_code_key(&address).hashed_key(); + let address_h256 = address_to_h256(&address); let Some(row) = sqlx::query!( r#" SELECT @@ -298,24 +300,31 @@ impl ContractVerificationDal<'_, '_> { FROM ( SELECT - * + miniblock_number, + tx_hash, + topic3 FROM - storage_logs + events WHERE - storage_logs.hashed_key = $1 - ORDER BY - miniblock_number DESC, - operation_number DESC + address = $1 + AND topic1 = $2 + AND topic4 = $3 LIMIT 1 - ) storage_logs - JOIN factory_deps ON factory_deps.bytecode_hash = storage_logs.value - LEFT JOIN transactions ON transactions.hash = storage_logs.tx_hash + ) deploy_event + JOIN factory_deps ON factory_deps.bytecode_hash = deploy_event.topic3 + LEFT JOIN transactions ON transactions.hash = deploy_event.tx_hash WHERE - storage_logs.value != $2 + deploy_event.miniblock_number <= ( + SELECT + MAX(number) + FROM + miniblocks + ) "#, - hashed_key.as_bytes(), - FAILED_CONTRACT_DEPLOYMENT_BYTECODE_HASH.as_bytes() + CONTRACT_DEPLOYER_ADDRESS.as_bytes(), + DEPLOY_EVENT_SIGNATURE.as_bytes(), + address_h256.as_bytes(), ) .fetch_optional(self.storage.conn()) .await? diff --git a/core/lib/dal/src/models/storage_log.rs b/core/lib/dal/src/models/storage_log.rs index cfbfe99bc679..ef3a018f9e46 100644 --- a/core/lib/dal/src/models/storage_log.rs +++ b/core/lib/dal/src/models/storage_log.rs @@ -16,7 +16,6 @@ pub struct DbStorageLog { pub key: H256, pub value: H256, pub operation_number: u64, - pub tx_hash: H256, pub l2_block_number: L2BlockNumber, } diff --git a/core/lib/dal/src/pruning_dal/tests.rs b/core/lib/dal/src/pruning_dal/tests.rs index 2670fe550c56..61b5766b93e7 100644 --- a/core/lib/dal/src/pruning_dal/tests.rs +++ b/core/lib/dal/src/pruning_dal/tests.rs @@ -291,7 +291,7 @@ async fn insert_l2_block_storage_logs( storage_logs: Vec, ) { conn.storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(H256::zero(), storage_logs)]) + .insert_storage_logs(l2_block_number, &storage_logs) .await .unwrap(); } diff --git a/core/lib/dal/src/snapshots_creator_dal.rs b/core/lib/dal/src/snapshots_creator_dal.rs index 7648efc43cc2..fef3ee5b7198 100644 --- a/core/lib/dal/src/snapshots_creator_dal.rs +++ b/core/lib/dal/src/snapshots_creator_dal.rs @@ -164,7 +164,7 @@ mod tests { logs.sort_unstable_by_key(|log| log.key.hashed_key()); conn.storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::zero(), logs.clone())]) + .insert_storage_logs(L2BlockNumber(1), &logs) .await .unwrap(); let mut written_keys: Vec<_> = logs.iter().map(|log| log.key).collect(); @@ -198,7 +198,7 @@ mod tests { let all_new_logs: Vec<_> = new_logs.chain(updated_logs).collect(); let all_new_logs_len = all_new_logs.len(); conn.storage_logs_dal() - .insert_storage_logs(L2BlockNumber(2), &[(H256::zero(), all_new_logs)]) + .insert_storage_logs(L2BlockNumber(2), &all_new_logs) .await .unwrap(); conn.storage_logs_dedup_dal() @@ -271,14 +271,14 @@ mod tests { StorageLog::new_write_log(key, H256::zero()), ]; conn.storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::zero(), phantom_writes)]) + .insert_storage_logs(L2BlockNumber(1), &phantom_writes) .await .unwrap(); // initial writes are intentionally not inserted. let real_write = StorageLog::new_write_log(key, H256::repeat_byte(2)); conn.storage_logs_dal() - .insert_storage_logs(L2BlockNumber(2), &[(H256::zero(), vec![real_write])]) + .insert_storage_logs(L2BlockNumber(2), &[real_write]) .await .unwrap(); conn.storage_logs_dedup_dal() diff --git a/core/lib/dal/src/storage_logs_dal.rs b/core/lib/dal/src/storage_logs_dal.rs index 7546812ae6c2..052e93370333 100644 --- a/core/lib/dal/src/storage_logs_dal.rs +++ b/core/lib/dal/src/storage_logs_dal.rs @@ -26,7 +26,7 @@ impl StorageLogsDal<'_, '_> { pub async fn insert_storage_logs( &mut self, block_number: L2BlockNumber, - logs: &[(H256, Vec)], + logs: &[StorageLog], ) -> DalResult<()> { self.insert_storage_logs_inner(block_number, logs, 0).await } @@ -34,13 +34,13 @@ impl StorageLogsDal<'_, '_> { async fn insert_storage_logs_inner( &mut self, block_number: L2BlockNumber, - logs: &[(H256, Vec)], + logs: &[StorageLog], mut operation_number: u32, ) -> DalResult<()> { let logs_len = logs.len(); let copy = CopyStatement::new( "COPY storage_logs( - hashed_key, address, key, value, operation_number, tx_hash, miniblock_number, + hashed_key, address, key, value, operation_number, miniblock_number, created_at, updated_at ) FROM STDIN WITH (DELIMITER '|')", @@ -53,23 +53,21 @@ impl StorageLogsDal<'_, '_> { let mut buffer = String::new(); let now = Utc::now().naive_utc().to_string(); - for (tx_hash, logs) in logs { - for log in logs { - write_str!( - &mut buffer, - r"\\x{hashed_key:x}|\\x{address:x}|\\x{key:x}|\\x{value:x}|", - hashed_key = log.key.hashed_key(), - address = log.key.address(), - key = log.key.key(), - value = log.value - ); - writeln_str!( - &mut buffer, - r"{operation_number}|\\x{tx_hash:x}|{block_number}|{now}|{now}" - ); - - operation_number += 1; - } + for log in logs { + write_str!( + &mut buffer, + r"\\x{hashed_key:x}|\\x{address:x}|\\x{key:x}|\\x{value:x}|", + hashed_key = log.key.hashed_key(), + address = log.key.address(), + key = log.key.key(), + value = log.value + ); + writeln_str!( + &mut buffer, + r"{operation_number}|{block_number}|{now}|{now}" + ); + + operation_number += 1; } copy.send(buffer.as_bytes()).await } @@ -117,7 +115,7 @@ impl StorageLogsDal<'_, '_> { pub async fn append_storage_logs( &mut self, block_number: L2BlockNumber, - logs: &[(H256, Vec)], + logs: &[StorageLog], ) -> DalResult<()> { let operation_number = sqlx::query!( r#" @@ -565,7 +563,6 @@ impl StorageLogsDal<'_, '_> { key, value, operation_number, - tx_hash, miniblock_number FROM storage_logs @@ -585,7 +582,6 @@ impl StorageLogsDal<'_, '_> { key: H256::from_slice(&row.key), value: H256::from_slice(&row.value), operation_number: row.operation_number as u64, - tx_hash: H256::from_slice(&row.tx_hash), l2_block_number: L2BlockNumber(row.miniblock_number as u32), }) .collect() @@ -745,7 +741,6 @@ mod tests { .await .unwrap(); - let logs = [(H256::zero(), logs)]; conn.storage_logs_dal() .insert_storage_logs(L2BlockNumber(number), &logs) .await @@ -783,7 +778,7 @@ mod tests { // Add more logs and check log ordering. let third_log = StorageLog::new_write_log(first_key, H256::repeat_byte(3)); - let more_logs = [(H256::repeat_byte(1), vec![third_log])]; + let more_logs = vec![third_log]; conn.storage_logs_dal() .append_storage_logs(L2BlockNumber(1), &more_logs) .await diff --git a/core/lib/dal/src/storage_logs_dedup_dal.rs b/core/lib/dal/src/storage_logs_dedup_dal.rs index 159f331a4753..6df54c54fc51 100644 --- a/core/lib/dal/src/storage_logs_dedup_dal.rs +++ b/core/lib/dal/src/storage_logs_dedup_dal.rs @@ -7,10 +7,9 @@ use zksync_db_connection::{ instrument::{CopyStatement, InstrumentExt}, }; use zksync_types::{ - snapshots::SnapshotStorageLog, zk_evm_types::LogQuery, AccountTreeId, Address, L1BatchNumber, - StorageKey, H256, + snapshots::SnapshotStorageLog, AccountTreeId, Address, L1BatchNumber, StorageKey, StorageLog, + H256, }; -use zksync_utils::u256_to_h256; pub use crate::models::storage_log::DbInitialWrite; use crate::Core; @@ -24,7 +23,7 @@ impl StorageLogsDedupDal<'_, '_> { pub async fn insert_protective_reads( &mut self, l1_batch_number: L1BatchNumber, - read_logs: &[LogQuery], + read_logs: &[StorageLog], ) -> DalResult<()> { let read_logs_len = read_logs.len(); let copy = CopyStatement::new( @@ -40,8 +39,8 @@ impl StorageLogsDedupDal<'_, '_> { let mut bytes: Vec = Vec::new(); let now = Utc::now().naive_utc().to_string(); for log in read_logs.iter() { - let address_str = format!("\\\\x{}", hex::encode(log.address.0)); - let key_str = format!("\\\\x{}", hex::encode(u256_to_h256(log.key).0)); + let address_str = format!("\\\\x{}", hex::encode(log.key.address())); + let key_str = format!("\\\\x{}", hex::encode(log.key.key())); let row = format!( "{}|{}|{}|{}|{}\n", l1_batch_number, address_str, key_str, now, now diff --git a/core/lib/dal/src/tokens_dal.rs b/core/lib/dal/src/tokens_dal.rs index cf0b89c950b2..b34b913c45c3 100644 --- a/core/lib/dal/src/tokens_dal.rs +++ b/core/lib/dal/src/tokens_dal.rs @@ -167,7 +167,6 @@ mod tests { .await .unwrap(); - let logs = [(H256::zero(), logs)]; conn.storage_logs_dal() .insert_storage_logs(L2BlockNumber(number), &logs) .await @@ -337,10 +336,7 @@ mod tests { ); storage .storage_logs_dal() - .insert_storage_logs( - L2BlockNumber(1), - &[(H256::zero(), vec![failed_deployment_log])], - ) + .insert_storage_logs(L2BlockNumber(1), &[failed_deployment_log]) .await .unwrap(); @@ -348,10 +344,7 @@ mod tests { StorageLog::new_write_log(get_code_key(&test_info.l2_address), H256::repeat_byte(2)); storage .storage_logs_dal() - .insert_storage_logs( - L2BlockNumber(100), - &[(H256::zero(), vec![test_deployment_log])], - ) + .insert_storage_logs(L2BlockNumber(100), &[test_deployment_log]) .await .unwrap(); storage diff --git a/core/lib/multivm/src/glue/types/vm/mod.rs b/core/lib/multivm/src/glue/types/vm/mod.rs index 47cddc2b8dd4..5cf5a61a9a7b 100644 --- a/core/lib/multivm/src/glue/types/vm/mod.rs +++ b/core/lib/multivm/src/glue/types/vm/mod.rs @@ -1,4 +1,5 @@ mod block_context_mode; +mod storage_log; mod storage_query; mod tx_execution_mode; mod tx_revert_reason; diff --git a/core/lib/multivm/src/glue/types/vm/storage_log.rs b/core/lib/multivm/src/glue/types/vm/storage_log.rs new file mode 100644 index 000000000000..322bc491e9ab --- /dev/null +++ b/core/lib/multivm/src/glue/types/vm/storage_log.rs @@ -0,0 +1,29 @@ +use zksync_types::{ + zk_evm_types::LogQuery, StorageLog, StorageLogQuery, StorageLogWithPreviousValue, +}; +use zksync_utils::u256_to_h256; + +use crate::glue::{GlueFrom, GlueInto}; + +impl> GlueFrom for StorageLog { + fn glue_from(value: T) -> Self { + StorageLog::from_log_query(&value.glue_into()) + } +} + +impl> GlueFrom for StorageLogWithPreviousValue { + fn glue_from(value: T) -> Self { + let query = value.glue_into(); + StorageLogWithPreviousValue { + log: StorageLog { + kind: query.log_type, + ..StorageLog::from_log_query(&query.log_query) + }, + previous_value: u256_to_h256(if query.log_query.rollback { + query.log_query.written_value + } else { + query.log_query.read_value + }), + } + } +} diff --git a/core/lib/multivm/src/glue/types/vm/vm_block_result.rs b/core/lib/multivm/src/glue/types/vm/vm_block_result.rs index 824acc1ddfd2..1ee9f5ea90f4 100644 --- a/core/lib/multivm/src/glue/types/vm/vm_block_result.rs +++ b/core/lib/multivm/src/glue/types/vm/vm_block_result.rs @@ -19,19 +19,18 @@ use crate::{ impl GlueFrom for crate::interface::FinishedL1Batch { fn glue_from(value: crate::vm_m5::vm_instance::VmBlockResult) -> Self { let storage_log_queries = value.full_result.storage_log_queries.clone(); - let deduplicated_storage_log_queries: Vec = - sort_storage_access_queries_1_3_3( - &storage_log_queries - .iter() - .map(|log| { - GlueInto::::glue_into(log.log_query) - }) - .collect_vec(), - ) - .1 - .into_iter() - .map(GlueInto::::glue_into) - .collect(); + let deduplicated_storage_logs: Vec = sort_storage_access_queries_1_3_3( + &storage_log_queries + .iter() + .map(|log| { + GlueInto::::glue_into(log.log_query) + }) + .collect_vec(), + ) + .1 + .into_iter() + .map(GlueInto::::glue_into) + .collect(); crate::interface::FinishedL1Batch { block_tip_execution_result: VmExecutionResultAndLogs { @@ -51,7 +50,7 @@ impl GlueFrom for crate::interface::Fi }, final_execution_state: CurrentExecutionState { events: value.full_result.events, - deduplicated_storage_log_queries: deduplicated_storage_log_queries + deduplicated_storage_logs: deduplicated_storage_logs .into_iter() .map(GlueInto::glue_into) .collect(), @@ -79,19 +78,18 @@ impl GlueFrom for crate::interface::Fi impl GlueFrom for crate::interface::FinishedL1Batch { fn glue_from(value: crate::vm_m6::vm_instance::VmBlockResult) -> Self { let storage_log_queries = value.full_result.storage_log_queries.clone(); - let deduplicated_storage_log_queries: Vec = - sort_storage_access_queries_1_3_3( - &storage_log_queries - .iter() - .map(|log| { - GlueInto::::glue_into(log.log_query) - }) - .collect_vec(), - ) - .1 - .into_iter() - .map(GlueInto::::glue_into) - .collect(); + let deduplicated_storage_logs: Vec = sort_storage_access_queries_1_3_3( + &storage_log_queries + .iter() + .map(|log| { + GlueInto::::glue_into(log.log_query) + }) + .collect_vec(), + ) + .1 + .into_iter() + .map(GlueInto::::glue_into) + .collect(); crate::interface::FinishedL1Batch { block_tip_execution_result: VmExecutionResultAndLogs { @@ -111,7 +109,7 @@ impl GlueFrom for crate::interface::Fi }, final_execution_state: CurrentExecutionState { events: value.full_result.events, - deduplicated_storage_log_queries: deduplicated_storage_log_queries + deduplicated_storage_logs: deduplicated_storage_logs .into_iter() .map(GlueInto::glue_into) .collect(), @@ -139,7 +137,7 @@ impl GlueFrom for crate::interface::Fi impl GlueFrom for crate::interface::FinishedL1Batch { fn glue_from(value: crate::vm_1_3_2::vm_instance::VmBlockResult) -> Self { let storage_log_queries = value.full_result.storage_log_queries.clone(); - let deduplicated_storage_log_queries = + let deduplicated_storage_logs = circuit_sequencer_api_1_3_3::sort_storage_access::sort_storage_access_queries( storage_log_queries.iter().map(|log| &log.log_query), ) @@ -169,7 +167,7 @@ impl GlueFrom for crate::interface: }, final_execution_state: CurrentExecutionState { events: value.full_result.events, - deduplicated_storage_log_queries: deduplicated_storage_log_queries + deduplicated_storage_logs: deduplicated_storage_logs .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/interface/types/outputs/execution_result.rs b/core/lib/multivm/src/interface/types/outputs/execution_result.rs index faa702f411b3..19ce9b599c80 100644 --- a/core/lib/multivm/src/interface/types/outputs/execution_result.rs +++ b/core/lib/multivm/src/interface/types/outputs/execution_result.rs @@ -3,7 +3,7 @@ use zksync_types::{ event::{extract_long_l2_to_l1_messages, extract_published_bytecodes}, l2_to_l1_log::{SystemL2ToL1Log, UserL2ToL1Log}, tx::ExecutionMetrics, - StorageLogQuery, Transaction, VmEvent, + StorageLogWithPreviousValue, Transaction, VmEvent, }; use zksync_utils::bytecode::bytecode_len_in_bytes; @@ -19,7 +19,7 @@ pub struct Refunds { /// Events/storage logs/l2->l1 logs created within transaction execution. #[derive(Debug, Clone, Default, PartialEq)] pub struct VmExecutionLogs { - pub storage_logs: Vec, + pub storage_logs: Vec, pub events: Vec, // For pre-boojum VMs, there was no distinction between user logs and system // logs and so all the outputted logs were treated as user_l2_to_l1_logs. diff --git a/core/lib/multivm/src/interface/types/outputs/execution_state.rs b/core/lib/multivm/src/interface/types/outputs/execution_state.rs index 581bf32bc453..cc7bb64d4030 100644 --- a/core/lib/multivm/src/interface/types/outputs/execution_state.rs +++ b/core/lib/multivm/src/interface/types/outputs/execution_state.rs @@ -1,7 +1,7 @@ use zksync_types::{ l2_to_l1_log::{SystemL2ToL1Log, UserL2ToL1Log}, zk_evm_types::LogQuery, - VmEvent, U256, + StorageLog, VmEvent, U256, }; /// State of the VM since the start of the batch execution. @@ -10,7 +10,7 @@ pub struct CurrentExecutionState { /// Events produced by the VM. pub events: Vec, /// The deduplicated storage logs produced by the VM. - pub deduplicated_storage_log_queries: Vec, + pub deduplicated_storage_logs: Vec, /// Hashes of the contracts used by the VM. pub used_contract_hashes: Vec, /// L2 to L1 logs produced by the VM. diff --git a/core/lib/multivm/src/versions/vm_1_3_2/oracles/storage.rs b/core/lib/multivm/src/versions/vm_1_3_2/oracles/storage.rs index 3b72f89fcbd6..692a0496751a 100644 --- a/core/lib/multivm/src/versions/vm_1_3_2/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_1_3_2/oracles/storage.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_3::{ }; use zksync_state::{StoragePtr, WriteStorage}; use zksync_types::{ - utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogQueryType, + utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -84,7 +84,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query, - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -100,9 +100,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; query.read_value = current_value; @@ -284,12 +284,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_1_3_2/utils.rs b/core/lib/multivm/src/versions/vm_1_3_2/utils.rs index bd07e256a443..0be7a2837af0 100644 --- a/core/lib/multivm/src/versions/vm_1_3_2/utils.rs +++ b/core/lib/multivm/src/versions/vm_1_3_2/utils.rs @@ -8,7 +8,7 @@ use zk_evm_1_3_3::{ use zksync_contracts::{read_zbin_bytecode, BaseSystemContracts}; use zksync_state::WriteStorage; use zksync_system_constants::ZKPORTER_IS_AVAILABLE; -use zksync_types::{Address, StorageLogQueryType, H160, MAX_L2_TX_GAS_LIMIT, U256}; +use zksync_types::{Address, StorageLogKind, H160, MAX_L2_TX_GAS_LIMIT, U256}; use zksync_utils::h256_to_u256; use crate::vm_1_3_2::{ @@ -257,5 +257,5 @@ pub(crate) fn calculate_computational_gas_used< #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } diff --git a/core/lib/multivm/src/versions/vm_1_3_2/vm.rs b/core/lib/multivm/src/versions/vm_1_3_2/vm.rs index 36ba32a8120f..5721f4e2185e 100644 --- a/core/lib/multivm/src/versions/vm_1_3_2/vm.rs +++ b/core/lib/multivm/src/versions/vm_1_3_2/vm.rs @@ -164,7 +164,7 @@ impl VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_1_4_1/implementation/logs.rs b/core/lib/multivm/src/versions/vm_1_4_1/implementation/logs.rs index c35dfd666f21..fb1b6f3625db 100644 --- a/core/lib/multivm/src/versions/vm_1_4_1/implementation/logs.rs +++ b/core/lib/multivm/src/versions/vm_1_4_1/implementation/logs.rs @@ -46,10 +46,7 @@ impl Vm { storage_logs_count + log_queries.len() + precompile_calls_count; VmExecutionLogs { - storage_logs: storage_logs - .into_iter() - .map(|log| log.glue_into()) - .collect(), + storage_logs: storage_logs.into_iter().map(GlueInto::glue_into).collect(), events, user_l2_to_l1_logs: user_logs .into_iter() diff --git a/core/lib/multivm/src/versions/vm_1_4_1/oracles/storage.rs b/core/lib/multivm/src/versions/vm_1_4_1/oracles/storage.rs index df0283c6bcc8..a5ff6b8883a6 100644 --- a/core/lib/multivm/src/versions/vm_1_4_1/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_1_4_1/oracles/storage.rs @@ -12,7 +12,7 @@ use zksync_types::{ compression::compress_with_best_strategy, BYTES_PER_DERIVED_KEY, BYTES_PER_ENUMERATION_INDEX, }, - AccountTreeId, Address, StorageKey, StorageLogQueryType, BOOTLOADER_ADDRESS, U256, + AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -139,7 +139,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query.glue_into(), - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -158,9 +158,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; self.set_initial_value(&key, current_value, query.timestamp); @@ -426,12 +426,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_1_4_1/utils/logs.rs b/core/lib/multivm/src/versions/vm_1_4_1/utils/logs.rs index 079c90a07fad..fab90d9bee56 100644 --- a/core/lib/multivm/src/versions/vm_1_4_1/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_1_4_1/utils/logs.rs @@ -1,6 +1,6 @@ use zk_evm_1_4_1::aux_structures::{LogQuery, Timestamp}; use zksync_state::WriteStorage; -use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogQueryType, VmEvent}; +use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogKind, VmEvent}; use crate::{ glue::GlueInto, @@ -33,5 +33,5 @@ pub(crate) fn collect_events_and_l1_system_logs_after_timestamp VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_1_4_2/implementation/logs.rs b/core/lib/multivm/src/versions/vm_1_4_2/implementation/logs.rs index 47acdfeb3bad..c307b7aa8099 100644 --- a/core/lib/multivm/src/versions/vm_1_4_2/implementation/logs.rs +++ b/core/lib/multivm/src/versions/vm_1_4_2/implementation/logs.rs @@ -46,10 +46,7 @@ impl Vm { storage_logs_count + log_queries.len() + precompile_calls_count; VmExecutionLogs { - storage_logs: storage_logs - .into_iter() - .map(|log| log.glue_into()) - .collect(), + storage_logs: storage_logs.into_iter().map(GlueInto::glue_into).collect(), events, user_l2_to_l1_logs: user_logs .into_iter() diff --git a/core/lib/multivm/src/versions/vm_1_4_2/oracles/storage.rs b/core/lib/multivm/src/versions/vm_1_4_2/oracles/storage.rs index e5cb044c5bdd..9cc9945f84ff 100644 --- a/core/lib/multivm/src/versions/vm_1_4_2/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_1_4_2/oracles/storage.rs @@ -12,7 +12,7 @@ use zksync_types::{ compression::compress_with_best_strategy, BYTES_PER_DERIVED_KEY, BYTES_PER_ENUMERATION_INDEX, }, - AccountTreeId, Address, StorageKey, StorageLogQueryType, BOOTLOADER_ADDRESS, U256, + AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -139,7 +139,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query.glue_into(), - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -158,9 +158,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; self.set_initial_value(&key, current_value, query.timestamp); @@ -426,12 +426,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_1_4_2/utils/logs.rs b/core/lib/multivm/src/versions/vm_1_4_2/utils/logs.rs index e5df3f5914ab..ef9f124773be 100644 --- a/core/lib/multivm/src/versions/vm_1_4_2/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_1_4_2/utils/logs.rs @@ -1,6 +1,6 @@ use zk_evm_1_4_1::aux_structures::{LogQuery, Timestamp}; use zksync_state::WriteStorage; -use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogQueryType, VmEvent}; +use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogKind, VmEvent}; use crate::{ glue::GlueInto, @@ -33,5 +33,5 @@ pub(crate) fn collect_events_and_l1_system_logs_after_timestamp VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/implementation/logs.rs b/core/lib/multivm/src/versions/vm_boojum_integration/implementation/logs.rs index 73be046d7978..daf077fcca51 100644 --- a/core/lib/multivm/src/versions/vm_boojum_integration/implementation/logs.rs +++ b/core/lib/multivm/src/versions/vm_boojum_integration/implementation/logs.rs @@ -48,10 +48,7 @@ impl Vm { storage_logs_count + log_queries.len() + precompile_calls_count; VmExecutionLogs { - storage_logs: storage_logs - .into_iter() - .map(|log| log.glue_into()) - .collect(), + storage_logs: storage_logs.into_iter().map(GlueInto::glue_into).collect(), events, user_l2_to_l1_logs: user_logs .into_iter() diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/oracles/storage.rs b/core/lib/multivm/src/versions/vm_boojum_integration/oracles/storage.rs index 1ea8c7822e18..e505c2d9630c 100644 --- a/core/lib/multivm/src/versions/vm_boojum_integration/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_boojum_integration/oracles/storage.rs @@ -12,7 +12,7 @@ use zksync_types::{ compression::compress_with_best_strategy, BYTES_PER_DERIVED_KEY, BYTES_PER_ENUMERATION_INDEX, }, - AccountTreeId, Address, StorageKey, StorageLogQueryType, BOOTLOADER_ADDRESS, U256, + AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -136,7 +136,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query, - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -155,9 +155,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; self.set_initial_value(&key, current_value, query.timestamp); @@ -423,12 +423,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_boojum_integration/utils/logs.rs b/core/lib/multivm/src/versions/vm_boojum_integration/utils/logs.rs index 1507f2d5e22d..bc15f88c5437 100644 --- a/core/lib/multivm/src/versions/vm_boojum_integration/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_boojum_integration/utils/logs.rs @@ -1,7 +1,7 @@ use zk_evm_1_3_3::aux_structures::LogQuery; use zk_evm_1_4_0::aux_structures::Timestamp; use zksync_state::WriteStorage; -use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogQueryType, VmEvent}; +use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogKind, VmEvent}; use crate::{ glue::GlueInto, @@ -34,5 +34,5 @@ pub(crate) fn collect_events_and_l1_system_logs_after_timestamp VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_latest/implementation/logs.rs b/core/lib/multivm/src/versions/vm_latest/implementation/logs.rs index 5ec6ef062b8a..b42ce16cd0f2 100644 --- a/core/lib/multivm/src/versions/vm_latest/implementation/logs.rs +++ b/core/lib/multivm/src/versions/vm_latest/implementation/logs.rs @@ -46,10 +46,7 @@ impl Vm { storage_logs_count + log_queries.len() + precompile_calls_count; VmExecutionLogs { - storage_logs: storage_logs - .into_iter() - .map(|log| log.glue_into()) - .collect(), + storage_logs: storage_logs.into_iter().map(GlueInto::glue_into).collect(), events, user_l2_to_l1_logs: user_logs .into_iter() diff --git a/core/lib/multivm/src/versions/vm_latest/oracles/storage.rs b/core/lib/multivm/src/versions/vm_latest/oracles/storage.rs index 42405414cd24..22503ce9881c 100644 --- a/core/lib/multivm/src/versions/vm_latest/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_latest/oracles/storage.rs @@ -16,7 +16,7 @@ use zksync_types::{ compression::compress_with_best_strategy, BYTES_PER_DERIVED_KEY, BYTES_PER_ENUMERATION_INDEX, }, - AccountTreeId, Address, StorageKey, StorageLogQueryType, BOOTLOADER_ADDRESS, U256, + AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::{h256_to_u256, u256_to_h256}; @@ -188,7 +188,7 @@ impl StorageOracle { fn record_storage_read(&mut self, query: LogQuery) { let storage_log_query = StorageLogQuery { log_query: query, - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }; self.storage_frames_stack @@ -206,9 +206,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; let mut storage_log_query = StorageLogQuery { @@ -498,12 +498,12 @@ impl VmStorageOracle for StorageOracle { .rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_latest/tests/l1_tx_execution.rs b/core/lib/multivm/src/versions/vm_latest/tests/l1_tx_execution.rs index 5a87ce59be2f..359190fc4787 100644 --- a/core/lib/multivm/src/versions/vm_latest/tests/l1_tx_execution.rs +++ b/core/lib/multivm/src/versions/vm_latest/tests/l1_tx_execution.rs @@ -98,7 +98,7 @@ fn test_l1_tx_execution() { let res = StorageWritesDeduplicator::apply_on_empty_state(&storage_logs); // Tx panicked - assert_eq!(res.initial_storage_writes - basic_initial_writes, 0); + assert_eq!(res.initial_storage_writes, basic_initial_writes); let tx = account.get_test_contract_transaction( deploy_tx.address, diff --git a/core/lib/multivm/src/versions/vm_latest/tests/refunds.rs b/core/lib/multivm/src/versions/vm_latest/tests/refunds.rs index 51b0cd2f4d9b..72d2271f7158 100644 --- a/core/lib/multivm/src/versions/vm_latest/tests/refunds.rs +++ b/core/lib/multivm/src/versions/vm_latest/tests/refunds.rs @@ -94,8 +94,8 @@ fn test_predetermined_refunded_gas() { ); assert_eq!( - current_state_with_predefined_refunds.deduplicated_storage_log_queries, - current_state_without_predefined_refunds.deduplicated_storage_log_queries + current_state_with_predefined_refunds.deduplicated_storage_logs, + current_state_without_predefined_refunds.deduplicated_storage_logs ); assert_eq!( current_state_with_predefined_refunds.used_contract_hashes, @@ -148,16 +148,16 @@ fn test_predetermined_refunded_gas() { assert_eq!( current_state_with_changed_predefined_refunds - .deduplicated_storage_log_queries + .deduplicated_storage_logs .len(), current_state_without_predefined_refunds - .deduplicated_storage_log_queries + .deduplicated_storage_logs .len() ); assert_ne!( - current_state_with_changed_predefined_refunds.deduplicated_storage_log_queries, - current_state_without_predefined_refunds.deduplicated_storage_log_queries + current_state_with_changed_predefined_refunds.deduplicated_storage_logs, + current_state_without_predefined_refunds.deduplicated_storage_logs ); assert_eq!( current_state_with_changed_predefined_refunds.used_contract_hashes, diff --git a/core/lib/multivm/src/versions/vm_latest/utils/logs.rs b/core/lib/multivm/src/versions/vm_latest/utils/logs.rs index 4deea36f09fb..82e096cd3e7f 100644 --- a/core/lib/multivm/src/versions/vm_latest/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_latest/utils/logs.rs @@ -1,6 +1,6 @@ use zk_evm_1_5_0::aux_structures::{LogQuery, Timestamp}; use zksync_state::WriteStorage; -use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogQueryType, VmEvent}; +use zksync_types::{l2_to_l1_log::L2ToL1Log, StorageLogKind, VmEvent}; use crate::{ glue::GlueInto, @@ -33,5 +33,5 @@ pub(crate) fn collect_events_and_l1_system_logs_after_timestamp VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_m5/oracles/storage.rs b/core/lib/multivm/src/versions/vm_m5/oracles/storage.rs index 02cf5e9cdbc0..7ccfdf2f30c7 100644 --- a/core/lib/multivm/src/versions/vm_m5/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_m5/oracles/storage.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_1::{ zkevm_opcode_defs::system_params::INITIAL_STORAGE_WRITE_PUBDATA_BYTES, }; use zksync_types::{ - utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogQueryType, + utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -94,7 +94,7 @@ impl StorageOracle { self.frames_stack.push_forward( StorageLogQuery { log_query: query, - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }, query.timestamp, ); @@ -109,9 +109,9 @@ impl StorageOracle { .write_to_storage(key, query.written_value, query.timestamp); let log_query_type = if self.storage.get_ptr().borrow_mut().is_write_initial(&key) { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; query.read_value = current_value; @@ -250,12 +250,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in rollbacks.iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_m5/utils.rs b/core/lib/multivm/src/versions/vm_m5/utils.rs index a4fc53c7ea4e..8c5bca674c69 100644 --- a/core/lib/multivm/src/versions/vm_m5/utils.rs +++ b/core/lib/multivm/src/versions/vm_m5/utils.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_1::{ }; use zksync_contracts::{read_zbin_bytecode, BaseSystemContracts}; use zksync_system_constants::ZKPORTER_IS_AVAILABLE; -use zksync_types::{Address, StorageLogQueryType, H160, MAX_L2_TX_GAS_LIMIT, U256}; +use zksync_types::{Address, StorageLogKind, H160, MAX_L2_TX_GAS_LIMIT, U256}; use zksync_utils::h256_to_u256; use crate::{ @@ -264,5 +264,5 @@ pub fn read_bootloader_test_code(test: &str) -> Vec { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } diff --git a/core/lib/multivm/src/versions/vm_m5/vm.rs b/core/lib/multivm/src/versions/vm_m5/vm.rs index e65978a9dc72..67bfec9b9703 100644 --- a/core/lib/multivm/src/versions/vm_m5/vm.rs +++ b/core/lib/multivm/src/versions/vm_m5/vm.rs @@ -175,7 +175,7 @@ impl VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduplicated_logs + deduplicated_storage_logs: deduplicated_logs .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_m6/oracles/storage.rs b/core/lib/multivm/src/versions/vm_m6/oracles/storage.rs index a354ef627e3a..5393b9e48169 100644 --- a/core/lib/multivm/src/versions/vm_m6/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_m6/oracles/storage.rs @@ -6,7 +6,7 @@ use zk_evm_1_3_1::{ zkevm_opcode_defs::system_params::INITIAL_STORAGE_WRITE_PUBDATA_BYTES, }; use zksync_types::{ - utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogQueryType, + utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -84,7 +84,7 @@ impl StorageOracle { self.frames_stack.push_forward( StorageLogQuery { log_query: query, - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }, query.timestamp, ); @@ -99,9 +99,9 @@ impl StorageOracle { .write_to_storage(key, query.written_value, query.timestamp); let log_query_type = if self.storage.get_ptr().borrow_mut().is_write_initial(&key) { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; query.read_value = current_value; @@ -254,12 +254,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_m6/utils.rs b/core/lib/multivm/src/versions/vm_m6/utils.rs index 9321b95d00f9..d9709022fe3c 100644 --- a/core/lib/multivm/src/versions/vm_m6/utils.rs +++ b/core/lib/multivm/src/versions/vm_m6/utils.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_1::{ }; use zksync_contracts::{read_zbin_bytecode, BaseSystemContracts}; use zksync_system_constants::ZKPORTER_IS_AVAILABLE; -use zksync_types::{Address, StorageLogQueryType, H160, MAX_L2_TX_GAS_LIMIT, U256}; +use zksync_types::{Address, StorageLogKind, H160, MAX_L2_TX_GAS_LIMIT, U256}; use zksync_utils::h256_to_u256; use crate::{ @@ -294,5 +294,5 @@ pub(crate) fn calculate_computational_gas_used< #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } diff --git a/core/lib/multivm/src/versions/vm_m6/vm.rs b/core/lib/multivm/src/versions/vm_m6/vm.rs index 8fd512ef575d..fe2deb4181a3 100644 --- a/core/lib/multivm/src/versions/vm_m6/vm.rs +++ b/core/lib/multivm/src/versions/vm_m6/vm.rs @@ -191,7 +191,7 @@ impl VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduplicated_logs + deduplicated_storage_logs: deduplicated_logs .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_refunds_enhancement/oracles/storage.rs b/core/lib/multivm/src/versions/vm_refunds_enhancement/oracles/storage.rs index 59ed4f9450e5..7b2cd8c61588 100644 --- a/core/lib/multivm/src/versions/vm_refunds_enhancement/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_refunds_enhancement/oracles/storage.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_3::{ }; use zksync_state::{StoragePtr, WriteStorage}; use zksync_types::{ - utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogQueryType, + utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -108,7 +108,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query.glue_into(), - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -124,9 +124,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; query.read_value = current_value; @@ -378,12 +378,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_refunds_enhancement/utils/logs.rs b/core/lib/multivm/src/versions/vm_refunds_enhancement/utils/logs.rs index ba1ed871f527..fc8b5bef62b3 100644 --- a/core/lib/multivm/src/versions/vm_refunds_enhancement/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_refunds_enhancement/utils/logs.rs @@ -1,9 +1,9 @@ use zk_evm_1_3_3::aux_structures::LogQuery; -use zksync_types::StorageLogQueryType; +use zksync_types::StorageLogKind; /// Log query, which handle initial and repeated writes to the storage #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } diff --git a/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs b/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs index f3d233d751a0..0bac1d7d47de 100644 --- a/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs +++ b/core/lib/multivm/src/versions/vm_refunds_enhancement/vm.rs @@ -108,7 +108,7 @@ impl VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/multivm/src/versions/vm_virtual_blocks/old_vm/oracles/storage.rs b/core/lib/multivm/src/versions/vm_virtual_blocks/old_vm/oracles/storage.rs index 423abfd1c4a2..682814b8d512 100644 --- a/core/lib/multivm/src/versions/vm_virtual_blocks/old_vm/oracles/storage.rs +++ b/core/lib/multivm/src/versions/vm_virtual_blocks/old_vm/oracles/storage.rs @@ -7,7 +7,7 @@ use zk_evm_1_3_3::{ }; use zksync_state::{StoragePtr, WriteStorage}; use zksync_types::{ - utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogQueryType, + utils::storage_key_for_eth_balance, AccountTreeId, Address, StorageKey, StorageLogKind, BOOTLOADER_ADDRESS, U256, }; use zksync_utils::u256_to_h256; @@ -87,7 +87,7 @@ impl StorageOracle { self.frames_stack.push_forward( Box::new(StorageLogQuery { log_query: query.glue_into(), - log_type: StorageLogQueryType::Read, + log_type: StorageLogKind::Read, }), query.timestamp, ); @@ -103,9 +103,9 @@ impl StorageOracle { let is_initial_write = self.storage.get_ptr().borrow_mut().is_write_initial(&key); let log_query_type = if is_initial_write { - StorageLogQueryType::InitialWrite + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; query.read_value = current_value; @@ -287,12 +287,12 @@ impl VmStorageOracle for StorageOracle { // perform actual rollback for query in self.frames_stack.rollback().current_frame().iter().rev() { let read_value = match query.log_type { - StorageLogQueryType::Read => { + StorageLogKind::Read => { // Having Read logs in rollback is not possible tracing::warn!("Read log in rollback queue {:?}", query); continue; } - StorageLogQueryType::InitialWrite | StorageLogQueryType::RepeatedWrite => { + StorageLogKind::InitialWrite | StorageLogKind::RepeatedWrite => { query.log_query.read_value } }; diff --git a/core/lib/multivm/src/versions/vm_virtual_blocks/utils/logs.rs b/core/lib/multivm/src/versions/vm_virtual_blocks/utils/logs.rs index ba1ed871f527..fc8b5bef62b3 100644 --- a/core/lib/multivm/src/versions/vm_virtual_blocks/utils/logs.rs +++ b/core/lib/multivm/src/versions/vm_virtual_blocks/utils/logs.rs @@ -1,9 +1,9 @@ use zk_evm_1_3_3::aux_structures::LogQuery; -use zksync_types::StorageLogQueryType; +use zksync_types::StorageLogKind; /// Log query, which handle initial and repeated writes to the storage #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } diff --git a/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs b/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs index 28883b91f110..ec9b12e82ed4 100644 --- a/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs +++ b/core/lib/multivm/src/versions/vm_virtual_blocks/vm.rs @@ -108,7 +108,7 @@ impl VmInterface for Vm { CurrentExecutionState { events, - deduplicated_storage_log_queries: deduped_storage_log_queries + deduplicated_storage_logs: deduped_storage_log_queries .into_iter() .map(GlueInto::glue_into) .collect(), diff --git a/core/lib/state/src/in_memory.rs b/core/lib/state/src/in_memory.rs index 73ceb4a59633..594eae128169 100644 --- a/core/lib/state/src/in_memory.rs +++ b/core/lib/state/src/in_memory.rs @@ -2,8 +2,8 @@ use std::collections::{hash_map::Entry, BTreeMap, HashMap}; use zksync_types::{ block::DeployedContract, get_code_key, get_known_code_key, get_system_context_init_logs, - system_contracts::get_system_smart_contracts, L2ChainId, StorageKey, StorageLog, - StorageLogKind, StorageValue, H256, U256, + system_contracts::get_system_smart_contracts, L2ChainId, StorageKey, StorageLog, StorageValue, + H256, U256, }; use zksync_utils::u256_to_h256; @@ -63,7 +63,7 @@ impl InMemoryStorage { ] }) .chain(system_context_init_log) - .filter_map(|log| (log.kind == StorageLogKind::Write).then_some((log.key, log.value))) + .filter_map(|log| (log.is_write()).then_some((log.key, log.value))) .collect(); let state: HashMap<_, _> = state_without_indices .into_iter() diff --git a/core/lib/state/src/test_utils.rs b/core/lib/state/src/test_utils.rs index 8a0b56588f30..52febc5040ee 100644 --- a/core/lib/state/src/test_utils.rs +++ b/core/lib/state/src/test_utils.rs @@ -94,7 +94,7 @@ pub(crate) async fn create_l2_block( .await .unwrap(); conn.storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(H256::zero(), block_logs)]) + .insert_storage_logs(l2_block_number, &block_logs) .await .unwrap(); } @@ -149,10 +149,7 @@ pub(crate) async fn prepare_postgres_for_snapshot_recovery( let snapshot_storage_logs = gen_storage_logs(100..200); conn.storage_logs_dal() - .insert_storage_logs( - snapshot_recovery.l2_block_number, - &[(H256::zero(), snapshot_storage_logs.clone())], - ) + .insert_storage_logs(snapshot_recovery.l2_block_number, &snapshot_storage_logs) .await .unwrap(); let mut written_keys: Vec<_> = snapshot_storage_logs.iter().map(|log| log.key).collect(); diff --git a/core/lib/tee_verifier/src/lib.rs b/core/lib/tee_verifier/src/lib.rs index 3828dc51201e..19e9c4655f40 100644 --- a/core/lib/tee_verifier/src/lib.rs +++ b/core/lib/tee_verifier/src/lib.rs @@ -21,11 +21,8 @@ use zksync_merkle_tree::{ use zksync_object_store::{serialize_using_bincode, Bucket, StoredObject}; use zksync_prover_interface::inputs::{PrepareBasicCircuitsJob, StorageLogMetadata}; use zksync_state::{InMemoryStorage, StorageView, WriteStorage}; -use zksync_types::{ - block::L2BlockExecutionData, ethabi::ethereum_types::BigEndianHash, zk_evm_types::LogQuery, - AccountTreeId, L1BatchNumber, StorageKey, H256, -}; -use zksync_utils::{bytecode::hash_bytecode, u256_to_h256}; +use zksync_types::{block::L2BlockExecutionData, L1BatchNumber, StorageLog, H256}; +use zksync_utils::bytecode::hash_bytecode; /// Version 1 of the data used as input for the TEE verifier. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -217,34 +214,30 @@ impl TeeVerifierInput { /// Map `LogQuery` and `TreeLogEntry` to a `TreeInstruction` fn map_log_tree( - log_query: &LogQuery, + storage_log: &StorageLog, tree_log_entry: &TreeLogEntry, idx: &mut u64, ) -> anyhow::Result { - let key = StorageKey::new( - AccountTreeId::new(log_query.address), - u256_to_h256(log_query.key), - ) - .hashed_key_u256(); - Ok(match (log_query.rw_flag, *tree_log_entry) { + let key = storage_log.key.hashed_key_u256(); + Ok(match (storage_log.is_write(), *tree_log_entry) { (true, TreeLogEntry::Updated { leaf_index, .. }) => { - TreeInstruction::write(key, leaf_index, H256(log_query.written_value.into())) + TreeInstruction::write(key, leaf_index, H256(storage_log.value.into())) } (true, TreeLogEntry::Inserted) => { let leaf_index = *idx; *idx += 1; - TreeInstruction::write(key, leaf_index, H256(log_query.written_value.into())) + TreeInstruction::write(key, leaf_index, H256(storage_log.value.into())) } (false, TreeLogEntry::Read { value, .. }) => { - if log_query.read_value != value.into_uint() { + if storage_log.value != value { tracing::error!( "Failed to map LogQuery to TreeInstruction: {:#?} != {:#?}", - log_query.read_value, + storage_log.value, value ); anyhow::bail!( "Failed to map LogQuery to TreeInstruction: {:#?} != {:#?}", - log_query.read_value, + storage_log.value, value ); } @@ -266,7 +259,7 @@ impl TeeVerifierInput { ) -> anyhow::Result> { vm_out .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .into_iter() .zip(bowp.logs.iter()) .map(|(log_query, tree_log_entry)| { diff --git a/core/lib/types/src/storage/log.rs b/core/lib/types/src/storage/log.rs index 6128f588668e..a05e25abccb5 100644 --- a/core/lib/types/src/storage/log.rs +++ b/core/lib/types/src/storage/log.rs @@ -2,19 +2,19 @@ use std::mem; use serde::{Deserialize, Serialize}; use zksync_basic_types::AccountTreeId; -use zksync_utils::u256_to_h256; +use zksync_utils::{h256_to_u256, u256_to_h256}; use crate::{ api::ApiStorageLog, - zk_evm_types::{LogQuery, Timestamp}, + zk_evm_types::{self, LogQuery, Timestamp}, StorageKey, StorageValue, U256, }; -// TODO (SMA-1269): Refactor `StorageLog/StorageLogQuery and StorageLogKind/StorageLongQueryType`. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] pub enum StorageLogKind { Read, - Write, + InitialWrite, + RepeatedWrite, } #[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] @@ -24,20 +24,23 @@ pub struct StorageLog { pub value: StorageValue, } +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct StorageLogWithPreviousValue { + pub log: StorageLog, + pub previous_value: StorageValue, +} + impl StorageLog { - pub fn from_log_query(log: &StorageLogQuery) -> Self { - let key = StorageKey::new( - AccountTreeId::new(log.log_query.address), - u256_to_h256(log.log_query.key), - ); - if log.log_query.rw_flag { - if log.log_query.rollback { - Self::new_write_log(key, u256_to_h256(log.log_query.read_value)) + pub fn from_log_query(log: &LogQuery) -> Self { + let key = StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)); + if log.rw_flag { + if log.rollback { + Self::new_write_log(key, u256_to_h256(log.read_value)) } else { - Self::new_write_log(key, u256_to_h256(log.log_query.written_value)) + Self::new_write_log(key, u256_to_h256(log.written_value)) } } else { - Self::new_read_log(key, u256_to_h256(log.log_query.read_value)) + Self::new_read_log(key, u256_to_h256(log.read_value)) } } @@ -51,12 +54,16 @@ impl StorageLog { pub fn new_write_log(key: StorageKey, value: StorageValue) -> Self { Self { - kind: StorageLogKind::Write, + kind: StorageLogKind::RepeatedWrite, key, value, } } + pub fn is_write(&self) -> bool { + !matches!(self.kind, StorageLogKind::Read) + } + /// Converts this log to a log query that could be used in tests. pub fn to_test_log_query(&self) -> LogQuery { let mut read_value = U256::zero(); @@ -74,33 +81,54 @@ impl StorageLog { key: U256::from_big_endian(self.key.key().as_bytes()), read_value, written_value, - rw_flag: matches!(self.kind, StorageLogKind::Write), + rw_flag: self.is_write(), rollback: false, is_service: false, } } } -#[derive(Debug, Clone, Copy, Eq, PartialEq, Serialize, Deserialize)] -pub enum StorageLogQueryType { - Read, - InitialWrite, - RepeatedWrite, +impl From for StorageLog { + fn from(log_query: zk_evm_types::LogQuery) -> Self { + Self::from_log_query(&log_query) + } +} + +impl From for ApiStorageLog { + fn from(storage_log: StorageLog) -> Self { + Self { + address: *storage_log.key.address(), + key: h256_to_u256(*storage_log.key.key()), + written_value: h256_to_u256(storage_log.value), + } + } +} + +impl From<&StorageLogWithPreviousValue> for ApiStorageLog { + fn from(log: &StorageLogWithPreviousValue) -> Self { + log.log.into() + } } /// Log query, which handle initial and repeated writes to the storage #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct StorageLogQuery { pub log_query: LogQuery, - pub log_type: StorageLogQueryType, + pub log_type: StorageLogKind, } impl From<&StorageLogQuery> for ApiStorageLog { fn from(log_query: &StorageLogQuery) -> Self { + log_query.log_query.into() + } +} + +impl From for ApiStorageLog { + fn from(log_query: LogQuery) -> Self { ApiStorageLog { - address: log_query.log_query.address, - key: log_query.log_query.key, - written_value: log_query.log_query.written_value, + address: log_query.address, + key: log_query.key, + written_value: log_query.written_value, } } } diff --git a/core/lib/types/src/storage_writes_deduplicator.rs b/core/lib/types/src/storage_writes_deduplicator.rs index a67686a7dc77..f9f3cc323b9f 100644 --- a/core/lib/types/src/storage_writes_deduplicator.rs +++ b/core/lib/types/src/storage_writes_deduplicator.rs @@ -1,19 +1,18 @@ use std::collections::HashMap; -use zksync_utils::u256_to_h256; +use zksync_basic_types::H256; +use zksync_utils::h256_to_u256; use crate::{ tx::tx_execution_info::DeduplicatedWritesMetrics, - writes::compression::compress_with_best_strategy, AccountTreeId, StorageKey, StorageLogQuery, - StorageLogQueryType, U256, + writes::compression::compress_with_best_strategy, StorageKey, StorageLogKind, + StorageLogWithPreviousValue, }; #[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct ModifiedSlot { /// Value of the slot after modification. - pub value: U256, - /// Index (in L1 batch) of the transaction that lastly modified the slot. - pub tx_index: u16, + pub value: H256, /// Size of pubdata update in bytes pub size: usize, } @@ -35,7 +34,7 @@ struct UpdateItem { /// Struct that allows to deduplicate storage writes in-flight. #[derive(Debug, Clone, PartialEq, Default)] pub struct StorageWritesDeduplicator { - initial_values: HashMap, + initial_values: HashMap, // stores the mapping of storage-slot key to its values and the tx number in block modified_key_values: HashMap, metrics: DeduplicatedWritesMetrics, @@ -55,13 +54,13 @@ impl StorageWritesDeduplicator { } /// Applies storage logs to the state. - pub fn apply<'a, I: IntoIterator>(&mut self, logs: I) { + pub fn apply<'a, I: IntoIterator>(&mut self, logs: I) { self.process_storage_logs(logs); } /// Returns metrics as if provided storage logs are applied to the state. /// It's implemented in the following way: apply logs -> save current metrics -> rollback logs. - pub fn apply_and_rollback<'a, I: IntoIterator>( + pub fn apply_and_rollback<'a, I: IntoIterator>( &mut self, logs: I, ) -> DeduplicatedWritesMetrics { @@ -72,7 +71,7 @@ impl StorageWritesDeduplicator { } /// Applies logs to the empty state and returns metrics. - pub fn apply_on_empty_state<'a, I: IntoIterator>( + pub fn apply_on_empty_state<'a, I: IntoIterator>( logs: I, ) -> DeduplicatedWritesMetrics { let mut deduplicator = Self::new(); @@ -83,29 +82,18 @@ impl StorageWritesDeduplicator { /// Processes storage logs and returns updates for `modified_keys` and `metrics` fields. /// Metrics can be used later to rollback the state. /// We don't care about `initial_values` changes as we only inserted values there and they are always valid. - fn process_storage_logs<'a, I: IntoIterator>( + fn process_storage_logs<'a, I: IntoIterator>( &mut self, logs: I, ) -> Vec { let mut updates = Vec::new(); - for log in logs.into_iter().filter(|log| log.log_query.rw_flag) { - let key = StorageKey::new( - AccountTreeId::new(log.log_query.address), - u256_to_h256(log.log_query.key), - ); - let initial_value = *self - .initial_values - .entry(key) - .or_insert(log.log_query.read_value); + for log in logs.into_iter().filter(|log| log.log.is_write()) { + let key = log.log.key; + let initial_value = *self.initial_values.entry(key).or_insert(log.previous_value); let was_key_modified = self.modified_key_values.contains_key(&key); - let modified_value = if log.log_query.rollback { - (initial_value != log.log_query.read_value).then_some(log.log_query.read_value) - } else { - (initial_value != log.log_query.written_value) - .then_some(log.log_query.written_value) - }; + let modified_value = (initial_value != log.log.value).then_some(log.log.value); - let is_write_initial = log.log_type == StorageLogQueryType::InitialWrite; + let is_write_initial = log.log.kind == StorageLogKind::InitialWrite; let field_to_change = if is_write_initial { &mut self.metrics.initial_storage_writes } else { @@ -113,7 +101,6 @@ impl StorageWritesDeduplicator { }; let total_size = &mut self.metrics.total_updated_values_size; - match (was_key_modified, modified_value) { (true, None) => { let value = self.modified_key_values.remove(&key).unwrap_or_else(|| { @@ -128,14 +115,17 @@ impl StorageWritesDeduplicator { }); } (true, Some(new_value)) => { - let value_size = compress_with_best_strategy(initial_value, new_value).len(); + let value_size = compress_with_best_strategy( + h256_to_u256(initial_value), + h256_to_u256(new_value), + ) + .len(); let old_value = self .modified_key_values .insert( key, ModifiedSlot { value: new_value, - tx_index: log.log_query.tx_number_in_block, size: value_size, }, ) @@ -153,12 +143,15 @@ impl StorageWritesDeduplicator { *total_size += value_size; } (false, Some(new_value)) => { - let value_size = compress_with_best_strategy(initial_value, new_value).len(); + let value_size = compress_with_best_strategy( + h256_to_u256(initial_value), + h256_to_u256(new_value), + ) + .len(); self.modified_key_values.insert( key, ModifiedSlot { value: new_value, - tx_index: log.log_query.tx_number_in_block, size: value_size, }, ); @@ -219,58 +212,59 @@ impl StorageWritesDeduplicator { #[cfg(test)] mod tests { + use zksync_basic_types::{AccountTreeId, U256}; + use zksync_utils::u256_to_h256; + use super::*; - use crate::{ - zk_evm_types::{LogQuery, Timestamp}, - H160, - }; + use crate::{StorageLog, H160}; - fn storage_log_query( + fn storage_log( key: U256, read_value: U256, written_value: U256, rollback: bool, is_initial: bool, - ) -> StorageLogQuery { - let log_type = if is_initial { - StorageLogQueryType::InitialWrite + ) -> StorageLogWithPreviousValue { + let kind = if is_initial { + StorageLogKind::InitialWrite } else { - StorageLogQueryType::RepeatedWrite + StorageLogKind::RepeatedWrite }; - StorageLogQuery { - log_query: LogQuery { - timestamp: Timestamp(0), - tx_number_in_block: 0, - aux_byte: 0, - shard_id: 0, - address: Default::default(), - key, - read_value, - written_value, - rw_flag: true, - rollback, - is_service: false, + StorageLogWithPreviousValue { + log: StorageLog { + key: StorageKey::new(AccountTreeId::default(), u256_to_h256(key)), + value: u256_to_h256(if rollback { read_value } else { written_value }), + kind, }, - log_type, + previous_value: u256_to_h256(if rollback { written_value } else { read_value }), } } - fn storage_log_query_with_address( + fn storage_log_with_address( address: H160, key: U256, written_value: U256, - ) -> StorageLogQuery { - let mut log = storage_log_query(key, 1234u32.into(), written_value, false, false); - log.log_query.address = address; - log + ) -> StorageLogWithPreviousValue { + StorageLogWithPreviousValue { + log: StorageLog { + key: StorageKey::new(AccountTreeId::new(address), u256_to_h256(key)), + value: u256_to_h256(written_value), + kind: StorageLogKind::RepeatedWrite, + }, + previous_value: H256::from_low_u64_be(1234), + } } #[test] fn storage_writes_deduplicator() { // Each test scenario is a tuple (input, expected output, description). - let scenarios: Vec<(Vec, DeduplicatedWritesMetrics, String)> = vec![ + let scenarios: Vec<( + Vec, + DeduplicatedWritesMetrics, + String, + )> = vec![ ( - vec![storage_log_query( + vec![storage_log( 0u32.into(), 0u32.into(), 1u32.into(), @@ -286,8 +280,8 @@ mod tests { ), ( vec![ - storage_log_query(0u32.into(), 0u32.into(), 1u32.into(), false, true), - storage_log_query(1u32.into(), 0u32.into(), 1u32.into(), false, false), + storage_log(0u32.into(), 0u32.into(), 1u32.into(), false, true), + storage_log(1u32.into(), 0u32.into(), 1u32.into(), false, false), ], DeduplicatedWritesMetrics { initial_storage_writes: 1, @@ -298,8 +292,8 @@ mod tests { ), ( vec![ - storage_log_query(0u32.into(), 0u32.into(), 1u32.into(), false, true), - storage_log_query(0u32.into(), 0u32.into(), 1u32.into(), true, true), + storage_log(0u32.into(), 0u32.into(), 1u32.into(), false, true), + storage_log(0u32.into(), 0u32.into(), 1u32.into(), true, true), ], DeduplicatedWritesMetrics { initial_storage_writes: 0, @@ -309,7 +303,7 @@ mod tests { "single rollback".into(), ), ( - vec![storage_log_query( + vec![storage_log( 0u32.into(), 10u32.into(), 10u32.into(), @@ -325,9 +319,9 @@ mod tests { ), ( vec![ - storage_log_query(0u32.into(), 0u32.into(), 1u32.into(), false, true), - storage_log_query(0u32.into(), 1u32.into(), 2u32.into(), false, true), - storage_log_query(0u32.into(), 2u32.into(), 0u32.into(), false, true), + storage_log(0u32.into(), 0u32.into(), 1u32.into(), false, true), + storage_log(0u32.into(), 1u32.into(), 2u32.into(), false, true), + storage_log(0u32.into(), 2u32.into(), 0u32.into(), false, true), ], DeduplicatedWritesMetrics { initial_storage_writes: 0, @@ -338,13 +332,13 @@ mod tests { ), ( vec![ - storage_log_query(0u32.into(), 5u32.into(), 10u32.into(), false, true), - storage_log_query(1u32.into(), 1u32.into(), 2u32.into(), false, true), - storage_log_query(0u32.into(), 10u32.into(), 11u32.into(), false, true), - storage_log_query(0u32.into(), 10u32.into(), 11u32.into(), true, true), - storage_log_query(2u32.into(), 0u32.into(), 10u32.into(), false, false), - storage_log_query(2u32.into(), 10u32.into(), 0u32.into(), false, false), - storage_log_query(2u32.into(), 0u32.into(), 10u32.into(), false, false), + storage_log(0u32.into(), 5u32.into(), 10u32.into(), false, true), + storage_log(1u32.into(), 1u32.into(), 2u32.into(), false, true), + storage_log(0u32.into(), 10u32.into(), 11u32.into(), false, true), + storage_log(0u32.into(), 10u32.into(), 11u32.into(), true, true), + storage_log(2u32.into(), 0u32.into(), 10u32.into(), false, false), + storage_log(2u32.into(), 10u32.into(), 0u32.into(), false, false), + storage_log(2u32.into(), 0u32.into(), 10u32.into(), false, false), ], DeduplicatedWritesMetrics { initial_storage_writes: 2, @@ -394,60 +388,54 @@ mod tests { ( new_storage_key(1, 5), ModifiedSlot { - value: 8u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(8), size: 2, }, ), ( new_storage_key(1, 4), ModifiedSlot { - value: 6u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(6), size: 2, }, ), ( new_storage_key(2, 5), ModifiedSlot { - value: 9u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(9), size: 2, }, ), ( new_storage_key(2, 4), ModifiedSlot { - value: 11u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(11), size: 2, }, ), ( new_storage_key(3, 5), ModifiedSlot { - value: 2u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(2), size: 2, }, ), ( new_storage_key(3, 4), ModifiedSlot { - value: 7u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(7), size: 2, }, ), ]); let mut deduplicator = StorageWritesDeduplicator::new(); let logs = [ - storage_log_query_with_address(H160::from_low_u64_be(1), 5u32.into(), 8u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(1), 4u32.into(), 6u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(2), 4u32.into(), 11u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(2), 5u32.into(), 9u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(3), 4u32.into(), 7u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(3), 5u32.into(), 2u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 5u32.into(), 8u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 4u32.into(), 6u32.into()), + storage_log_with_address(H160::from_low_u64_be(2), 4u32.into(), 11u32.into()), + storage_log_with_address(H160::from_low_u64_be(2), 5u32.into(), 9u32.into()), + storage_log_with_address(H160::from_low_u64_be(3), 4u32.into(), 7u32.into()), + storage_log_with_address(H160::from_low_u64_be(3), 5u32.into(), 2u32.into()), ]; deduplicator.apply(&logs); assert_eq!(expected, deduplicator.modified_key_values); @@ -459,36 +447,33 @@ mod tests { ( new_storage_key(1, 5), ModifiedSlot { - value: 6u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(6), size: 2, }, ), ( new_storage_key(2, 4), ModifiedSlot { - value: 11u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(11), size: 2, }, ), ( new_storage_key(3, 6), ModifiedSlot { - value: 7u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(7), size: 2, }, ), ]); let mut deduplicator = StorageWritesDeduplicator::new(); let logs = [ - storage_log_query_with_address(H160::from_low_u64_be(1), 5u32.into(), 8u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(1), 5u32.into(), 6u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(2), 4u32.into(), 9u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(2), 4u32.into(), 11u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(3), 6u32.into(), 2u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(3), 6u32.into(), 7u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 5u32.into(), 8u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 5u32.into(), 6u32.into()), + storage_log_with_address(H160::from_low_u64_be(2), 4u32.into(), 9u32.into()), + storage_log_with_address(H160::from_low_u64_be(2), 4u32.into(), 11u32.into()), + storage_log_with_address(H160::from_low_u64_be(3), 6u32.into(), 2u32.into()), + storage_log_with_address(H160::from_low_u64_be(3), 6u32.into(), 7u32.into()), ]; deduplicator.apply(&logs); assert_eq!(expected, deduplicator.modified_key_values); @@ -500,33 +485,30 @@ mod tests { ( new_storage_key(1, 2), ModifiedSlot { - value: 3u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(3), size: 2, }, ), ( new_storage_key(1, 2), ModifiedSlot { - value: 4u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(4), size: 2, }, ), ( new_storage_key(1, 2), ModifiedSlot { - value: 5u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(5), size: 2, }, ), ]); let mut deduplicator = StorageWritesDeduplicator::new(); let logs = [ - storage_log_query_with_address(H160::from_low_u64_be(1), 2u32.into(), 3u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(1), 2u32.into(), 4u32.into()), - storage_log_query_with_address(H160::from_low_u64_be(1), 2u32.into(), 5u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 2u32.into(), 3u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 2u32.into(), 4u32.into()), + storage_log_with_address(H160::from_low_u64_be(1), 2u32.into(), 5u32.into()), ]; deduplicator.apply(&logs); assert_eq!(expected, deduplicator.modified_key_values); @@ -537,28 +519,27 @@ mod tests { let expected = HashMap::from([( new_storage_key(0, 1), ModifiedSlot { - value: 2u32.into(), - tx_index: 0, + value: H256::from_low_u64_be(2), size: 2, }, )]); let mut deduplicator = StorageWritesDeduplicator::new(); let logs = [ - storage_log_query( + storage_log( U256::from(1u32), U256::from(1u32), U256::from(2u32), false, false, ), - storage_log_query( + storage_log( U256::from(1u32), U256::from(2u32), U256::from(1u32), false, false, ), - storage_log_query( + storage_log( U256::from(1u32), U256::from(2u32), U256::from(1u32), diff --git a/core/node/api_server/src/tx_sender/proxy.rs b/core/node/api_server/src/tx_sender/proxy.rs index 52fcc8a1a8b0..e179cdcb7748 100644 --- a/core/node/api_server/src/tx_sender/proxy.rs +++ b/core/node/api_server/src/tx_sender/proxy.rs @@ -609,7 +609,7 @@ mod tests { let nonce_log = StorageLog::new_write_log(nonce_key, H256::from_low_u64_be(1)); storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::zero(), vec![nonce_log])]) + .insert_storage_logs(L2BlockNumber(1), &[nonce_log]) .await .unwrap(); @@ -698,7 +698,7 @@ mod tests { let nonce_log = StorageLog::new_write_log(nonce_key, H256::from_low_u64_be(1)); storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::zero(), vec![nonce_log])]) + .insert_storage_logs(L2BlockNumber(1), &[nonce_log]) .await .unwrap(); diff --git a/core/node/api_server/src/tx_sender/tests.rs b/core/node/api_server/src/tx_sender/tests.rs index 897808447e7c..154e94280f36 100644 --- a/core/node/api_server/src/tx_sender/tests.rs +++ b/core/node/api_server/src/tx_sender/tests.rs @@ -27,7 +27,7 @@ async fn getting_nonce_for_account() { let nonce_log = StorageLog::new_write_log(nonce_key, H256::from_low_u64_be(123)); storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::default(), vec![nonce_log])]) + .append_storage_logs(L2BlockNumber(0), &[nonce_log]) .await .unwrap(); @@ -49,7 +49,7 @@ async fn getting_nonce_for_account() { }; storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::default(), vec![nonce_log])]) + .insert_storage_logs(L2BlockNumber(1), &[nonce_log]) .await .unwrap(); @@ -95,10 +95,7 @@ async fn getting_nonce_for_account_after_snapshot_recovery() { )]; storage .storage_logs_dal() - .insert_storage_logs( - SNAPSHOT_L2_BLOCK_NUMBER + 1, - &[(H256::default(), new_nonce_logs)], - ) + .insert_storage_logs(SNAPSHOT_L2_BLOCK_NUMBER + 1, &new_nonce_logs) .await .unwrap(); @@ -134,7 +131,7 @@ async fn submitting_tx_requires_one_connection() { let storage_log = StorageLog::new_write_log(balance_key, u256_to_h256(U256::one() << 64)); storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![storage_log])]) + .append_storage_logs(L2BlockNumber(0), &[storage_log]) .await .unwrap(); drop(storage); diff --git a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs index 5a4f7eb1f5fa..45cb312dde6e 100644 --- a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs +++ b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs @@ -10,7 +10,7 @@ use zksync_types::{ fee_model::{FeeParams, PubdataIndependentBatchFeeModelInput}, transaction_request::CallRequest, web3::Bytes, - Address, L1BatchNumber, L2BlockNumber, StorageLogQueryType, H256, U256, U64, + Address, L1BatchNumber, L2BlockNumber, H256, U256, U64, }; use zksync_web3_decl::{ jsonrpsee::core::{async_trait, RpcResult}, @@ -198,7 +198,7 @@ impl ZksNamespaceServer for ZksNamespace { .logs .storage_logs .iter() - .filter(|x| x.log_type != StorageLogQueryType::Read) + .filter(|x| x.log.is_write()) .map(ApiStorageLog::from) .collect_vec(), events: result diff --git a/core/node/api_server/src/web3/tests/mod.rs b/core/node/api_server/src/web3/tests/mod.rs index b9e8c96a3b1b..b2331a547707 100644 --- a/core/node/api_server/src/web3/tests/mod.rs +++ b/core/node/api_server/src/web3/tests/mod.rs @@ -673,7 +673,7 @@ impl HttpTest for TransactionCountTest { ); storage .storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(H256::zero(), vec![nonce_log])]) + .insert_storage_logs(l2_block_number, &[nonce_log]) .await?; } @@ -887,7 +887,7 @@ impl HttpTest for AllAccountBalancesTest { let eth_balance_log = StorageLog::new_write_log(eth_balance_key, u256_to_h256(eth_balance)); storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(1), &[(H256::zero(), vec![eth_balance_log])]) + .insert_storage_logs(L2BlockNumber(1), &[eth_balance_log]) .await?; // Create a custom token, but don't set balance for it yet. let custom_token = TokenInfo { @@ -913,7 +913,7 @@ impl HttpTest for AllAccountBalancesTest { StorageLog::new_write_log(token_balance_key, u256_to_h256(token_balance)); storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(2), &[(H256::zero(), vec![token_balance_log])]) + .insert_storage_logs(L2BlockNumber(2), &[token_balance_log]) .await?; let balances = client.get_all_account_balances(Self::ADDRESS).await?; diff --git a/core/node/api_server/src/web3/tests/vm.rs b/core/node/api_server/src/web3/tests/vm.rs index 372d9f35dd97..cb59f2f88e25 100644 --- a/core/node/api_server/src/web3/tests/vm.rs +++ b/core/node/api_server/src/web3/tests/vm.rs @@ -11,8 +11,8 @@ use zksync_types::{ api::{ApiStorageLog, Log}, get_intrinsic_constants, transaction_request::CallRequest, - zk_evm_types::{LogQuery, Timestamp}, - K256PrivateKey, L2ChainId, PackedEthSignature, StorageLogQuery, StorageLogQueryType, U256, + K256PrivateKey, L2ChainId, PackedEthSignature, StorageLogKind, StorageLogWithPreviousValue, + U256, }; use zksync_utils::u256_to_h256; use zksync_web3_decl::namespaces::DebugNamespaceClient; @@ -239,10 +239,7 @@ impl HttpTest for SendRawTransactionTest { let mut storage = pool.connection().await?; storage .storage_logs_dal() - .append_storage_logs( - L2BlockNumber(0), - &[(H256::zero(), vec![Self::balance_storage_log()])], - ) + .append_storage_logs(L2BlockNumber(0), &[Self::balance_storage_log()]) .await?; } @@ -273,40 +270,35 @@ async fn send_raw_transaction_after_snapshot_recovery() { struct SendTransactionWithDetailedOutputTest; impl SendTransactionWithDetailedOutputTest { - fn storage_logs(&self) -> Vec { - let log_query = LogQuery { - timestamp: Timestamp(100), - tx_number_in_block: 1, - aux_byte: 1, - shard_id: 2, - address: Address::zero(), - key: U256::one(), - read_value: U256::one(), - written_value: U256::one(), - rw_flag: false, - rollback: false, - is_service: false, + fn storage_logs(&self) -> Vec { + let log = StorageLog { + key: StorageKey::new( + AccountTreeId::new(Address::zero()), + u256_to_h256(U256::one()), + ), + value: u256_to_h256(U256::one()), + kind: StorageLogKind::Read, }; - vec![ - StorageLogQuery { - log_query, - log_type: StorageLogQueryType::Read, + [ + StorageLog { + kind: StorageLogKind::Read, + ..log }, - StorageLogQuery { - log_query: LogQuery { - tx_number_in_block: 2, - ..log_query - }, - log_type: StorageLogQueryType::InitialWrite, + StorageLog { + kind: StorageLogKind::InitialWrite, + ..log }, - StorageLogQuery { - log_query: LogQuery { - tx_number_in_block: 3, - ..log_query - }, - log_type: StorageLogQueryType::RepeatedWrite, + StorageLog { + kind: StorageLogKind::RepeatedWrite, + ..log }, ] + .into_iter() + .map(|log| StorageLogWithPreviousValue { + log, + previous_value: u256_to_h256(U256::one()), + }) + .collect() } fn vm_events(&self) -> Vec { @@ -356,10 +348,7 @@ impl HttpTest for SendTransactionWithDetailedOutputTest { .storage_logs_dal() .append_storage_logs( L2BlockNumber(0), - &[( - H256::zero(), - vec![SendRawTransactionTest::balance_storage_log()], - )], + &[SendRawTransactionTest::balance_storage_log()], ) .await?; @@ -383,7 +372,7 @@ impl HttpTest for SendTransactionWithDetailedOutputTest { send_result.storage_logs, self.storage_logs() .iter() - .filter(|x| x.log_type != StorageLogQueryType::Read) + .filter(|x| x.log.is_write()) .map(ApiStorageLog::from) .collect_vec() ); @@ -609,7 +598,7 @@ impl HttpTest for EstimateGasTest { let mut storage = pool.connection().await?; storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![storage_log])]) + .append_storage_logs(L2BlockNumber(0), &[storage_log]) .await?; } let mut call_request = CallRequest::from(l2_transaction); diff --git a/core/node/block_reverter/src/tests.rs b/core/node/block_reverter/src/tests.rs index 0fb54bdb1f93..7b989574b094 100644 --- a/core/node/block_reverter/src/tests.rs +++ b/core/node/block_reverter/src/tests.rs @@ -96,10 +96,7 @@ async fn setup_storage(storage: &mut Connection<'_, Core>, storage_logs: &[Stora storage .storage_logs_dal() - .insert_storage_logs( - l2_block_header.number, - &[(H256::zero(), vec![*storage_log])], - ) + .insert_storage_logs(l2_block_header.number, &[*storage_log]) .await .unwrap(); storage diff --git a/core/node/commitment_generator/src/tests.rs b/core/node/commitment_generator/src/tests.rs index 7f3c3eb2e2b1..29f17fa1646f 100644 --- a/core/node/commitment_generator/src/tests.rs +++ b/core/node/commitment_generator/src/tests.rs @@ -26,7 +26,7 @@ async fn seal_l1_batch(storage: &mut Connection<'_, Core>, number: L1BatchNumber let storage_log = StorageLog::new_write_log(storage_key, H256::repeat_byte(0xff)); storage .storage_logs_dal() - .insert_storage_logs(l2_block.number, &[(H256::zero(), vec![storage_log])]) + .insert_storage_logs(l2_block.number, &[storage_log]) .await .unwrap(); storage diff --git a/core/node/genesis/src/utils.rs b/core/node/genesis/src/utils.rs index cc5abd18cd58..7fdbe05da368 100644 --- a/core/node/genesis/src/utils.rs +++ b/core/node/genesis/src/utils.rs @@ -14,8 +14,7 @@ use zksync_types::{ get_code_key, get_known_code_key, get_system_context_init_logs, tokens::{TokenInfo, TokenMetadata}, zk_evm_types::{LogQuery, Timestamp}, - AccountTreeId, L1BatchNumber, L2BlockNumber, L2ChainId, StorageKey, StorageLog, StorageLogKind, - H256, + AccountTreeId, L1BatchNumber, L2BlockNumber, L2ChainId, StorageKey, StorageLog, H256, }; use zksync_utils::{be_words_to_bytes, bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; @@ -41,15 +40,12 @@ pub(super) async fn add_eth_token(transaction: &mut Connection<'_, Core>) -> any Ok(()) } -pub(super) fn get_storage_logs( - system_contracts: &[DeployedContract], -) -> Vec<(H256, Vec)> { - let system_context_init_logs = ( - H256::default(), +pub(super) fn get_storage_logs(system_contracts: &[DeployedContract]) -> Vec { + let system_context_init_logs = // During the genesis all chains have the same id. // TODO(EVM-579): make sure that the logic is compatible with Era. - get_system_context_init_logs(L2ChainId::from(DEFAULT_ERA_CHAIN_ID)), - ); + get_system_context_init_logs(L2ChainId::from(DEFAULT_ERA_CHAIN_ID)) + ; let known_code_storage_logs: Vec<_> = system_contracts .iter() @@ -57,15 +53,10 @@ pub(super) fn get_storage_logs( let hash = hash_bytecode(&contract.bytecode); let known_code_key = get_known_code_key(&hash); let marked_known_value = H256::from_low_u64_be(1u64); - ( - H256::default(), - vec![StorageLog::new_write_log( - known_code_key, - marked_known_value, - )], - ) + + StorageLog::new_write_log(known_code_key, marked_known_value) }) - .dedup_by(|a, b| a.1 == b.1) + .dedup_by(|a, b| a == b) .collect(); let storage_logs: Vec<_> = system_contracts @@ -73,46 +64,37 @@ pub(super) fn get_storage_logs( .map(|contract| { let hash = hash_bytecode(&contract.bytecode); let code_key = get_code_key(contract.account_id.address()); - ( - H256::default(), - vec![StorageLog::new_write_log(code_key, hash)], - ) + StorageLog::new_write_log(code_key, hash) }) - .chain(Some(system_context_init_logs)) + .chain(system_context_init_logs) .chain(known_code_storage_logs) .collect(); storage_logs } -pub(super) fn get_deduped_log_queries(storage_logs: &[(H256, Vec)]) -> Vec { +pub(super) fn get_deduped_log_queries(storage_logs: &[StorageLog]) -> Vec { // we don't produce proof for the genesis block, // but we still need to populate the table // to have the correct initial state of the merkle tree let log_queries: Vec = storage_logs .iter() - .enumerate() - .flat_map(|(tx_index, (_, storage_logs))| { - storage_logs - .iter() - .enumerate() - .map(move |(log_index, storage_log)| { - MultiVmLogQuery { - // Monotonically increasing Timestamp. Normally it's generated by the VM, but we don't have a VM in the genesis block. - timestamp: MultiVMTimestamp(((tx_index << 16) + log_index) as u32), - tx_number_in_block: tx_index as u16, - aux_byte: 0, - shard_id: 0, - address: *storage_log.key.address(), - key: h256_to_u256(*storage_log.key.key()), - read_value: h256_to_u256(H256::zero()), - written_value: h256_to_u256(storage_log.value), - rw_flag: storage_log.kind == StorageLogKind::Write, - rollback: false, - is_service: false, - } - }) - .collect::>() + .map(move |storage_log| { + MultiVmLogQuery { + // Timestamp and `tx_number` in block don't matter. + // `sort_storage_access_queries` assumes that the queries are in chronological order. + timestamp: MultiVMTimestamp(0), + tx_number_in_block: 0, + aux_byte: 0, + shard_id: 0, + address: *storage_log.key.address(), + key: h256_to_u256(*storage_log.key.key()), + read_value: h256_to_u256(H256::zero()), + written_value: h256_to_u256(storage_log.value), + rw_flag: storage_log.is_write(), + rollback: false, + is_service: false, + } }) .collect(); @@ -191,7 +173,7 @@ pub(super) async fn save_genesis_l1_batch_metadata( pub(super) async fn insert_system_contracts( storage: &mut Connection<'_, Core>, factory_deps: HashMap>, - storage_logs: &[(H256, Vec)], + storage_logs: &[StorageLog], ) -> Result<(), GenesisError> { let (deduplicated_writes, protective_reads): (Vec<_>, Vec<_>) = get_deduped_log_queries(storage_logs) @@ -206,7 +188,13 @@ pub(super) async fn insert_system_contracts( transaction .storage_logs_dedup_dal() - .insert_protective_reads(L1BatchNumber(0), &protective_reads) + .insert_protective_reads( + L1BatchNumber(0), + &protective_reads + .into_iter() + .map(StorageLog::from) + .collect::>(), + ) .await?; let written_storage_keys: Vec<_> = deduplicated_writes diff --git a/core/node/metadata_calculator/src/helpers.rs b/core/node/metadata_calculator/src/helpers.rs index 5f046a0d8b0d..d6918b7a5e87 100644 --- a/core/node/metadata_calculator/src/helpers.rs +++ b/core/node/metadata_calculator/src/helpers.rs @@ -1087,11 +1087,7 @@ mod tests { let mut logs = gen_storage_logs(100..120, 1); let logs_copy = logs[0].clone(); logs.push(logs_copy); - let read_logs: Vec<_> = logs[1] - .iter() - .step_by(3) - .map(StorageLog::to_test_log_query) - .collect(); + let read_logs: Vec<_> = logs[1].iter().step_by(3).cloned().collect(); extend_db_state(&mut storage, logs).await; storage .storage_logs_dedup_dal() diff --git a/core/node/metadata_calculator/src/tests.rs b/core/node/metadata_calculator/src/tests.rs index 20a814630fa7..fbdfe6cab322 100644 --- a/core/node/metadata_calculator/src/tests.rs +++ b/core/node/metadata_calculator/src/tests.rs @@ -682,7 +682,7 @@ pub(super) async fn extend_db_state_from_l1_batch( .unwrap(); storage .storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(H256::zero(), batch_logs)]) + .insert_storage_logs(l2_block_number, &batch_logs) .await .unwrap(); storage diff --git a/core/node/state_keeper/src/batch_executor/tests/tester.rs b/core/node/state_keeper/src/batch_executor/tests/tester.rs index 39f860b752e7..7e734ffc3d5e 100644 --- a/core/node/state_keeper/src/batch_executor/tests/tester.rs +++ b/core/node/state_keeper/src/batch_executor/tests/tester.rs @@ -272,7 +272,7 @@ impl Tester { storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![storage_log])]) + .append_storage_logs(L2BlockNumber(0), &[storage_log]) .await .unwrap(); if storage @@ -487,7 +487,7 @@ impl StorageSnapshot { if let TxExecutionResult::Success { tx_result, .. } = res { let storage_logs = &tx_result.logs.storage_logs; storage_writes_deduplicator - .apply(storage_logs.iter().filter(|log| log.log_query.rw_flag)); + .apply(storage_logs.iter().filter(|log| log.log.is_write())); } else { panic!("Unexpected tx execution result: {res:?}"); }; @@ -507,12 +507,12 @@ impl StorageSnapshot { let finished_batch = executor.finish_batch().await.unwrap(); let storage_logs = &finished_batch.block_tip_execution_result.logs.storage_logs; - storage_writes_deduplicator.apply(storage_logs.iter().filter(|log| log.log_query.rw_flag)); + storage_writes_deduplicator.apply(storage_logs.iter().filter(|log| log.log.is_write())); let modified_entries = storage_writes_deduplicator.into_modified_key_values(); all_logs.extend( modified_entries .into_iter() - .map(|(key, slot)| (key, u256_to_h256(slot.value))), + .map(|(key, slot)| (key, slot.value)), ); // Compute the hash of the last (fictive) L2 block in the batch. diff --git a/core/node/state_keeper/src/io/persistence.rs b/core/node/state_keeper/src/io/persistence.rs index 25b1ae9e6ea4..c3da618fe76f 100644 --- a/core/node/state_keeper/src/io/persistence.rs +++ b/core/node/state_keeper/src/io/persistence.rs @@ -4,11 +4,10 @@ use std::{sync::Arc, time::Instant}; use anyhow::Context as _; use async_trait::async_trait; -use multivm::zk_evm_latest::ethereum_types::H256; use tokio::sync::{mpsc, oneshot}; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_shared_metrics::{BlockStage, APP_METRICS}; -use zksync_types::{writes::TreeWrite, AccountTreeId, Address, StorageKey}; +use zksync_types::{writes::TreeWrite, Address}; use zksync_utils::u256_to_h256; use crate::{ @@ -306,17 +305,12 @@ impl StateKeeperOutputHandler for TreeWritesPersistence { } else { let deduplicated_writes = finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .iter() - .filter(|log_query| log_query.rw_flag); + .filter(|log_query| log_query.is_write()); let deduplicated_writes_hashed_keys: Vec<_> = deduplicated_writes .clone() - .map(|log| { - H256(StorageKey::raw_hashed_key( - &log.address, - &u256_to_h256(log.key), - )) - }) + .map(|log| log.key.hashed_key()) .collect(); let non_initial_writes = connection .storage_logs_dal() @@ -324,19 +318,18 @@ impl StateKeeperOutputHandler for TreeWritesPersistence { .await?; deduplicated_writes .map(|log| { - let key = - StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)); - let leaf_index = - if let Some((_, leaf_index)) = non_initial_writes.get(&key.hashed_key()) { - *leaf_index - } else { - next_index += 1; - next_index - 1 - }; + let leaf_index = if let Some((_, leaf_index)) = + non_initial_writes.get(&log.key.hashed_key()) + { + *leaf_index + } else { + next_index += 1; + next_index - 1 + }; TreeWrite { - address: log.address, - key: u256_to_h256(log.key), - value: u256_to_h256(log.written_value), + address: *log.key.address(), + key: *log.key.key(), + value: log.value, leaf_index, } }) @@ -363,10 +356,9 @@ mod tests { use zksync_node_genesis::{insert_genesis_batch, GenesisParams}; use zksync_types::{ api::TransactionStatus, block::BlockGasCount, tx::ExecutionMetrics, - writes::StateDiffRecord, AccountTreeId, L1BatchNumber, L2BlockNumber, StorageKey, - StorageLogQueryType, + writes::StateDiffRecord, L1BatchNumber, L2BlockNumber, StorageLogKind, }; - use zksync_utils::u256_to_h256; + use zksync_utils::h256_to_u256; use super::*; use crate::{ @@ -465,7 +457,7 @@ mod tests { (U256::from(1), Query::Read(U256::from(0))), (U256::from(2), Query::InitialWrite(U256::from(1))), ]; - let tx_result = create_execution_result(0, storage_logs); + let tx_result = create_execution_result(storage_logs); let storage_logs = tx_result.logs.storage_logs.clone(); updates.extend_from_executed_transaction( tx, @@ -482,27 +474,19 @@ mod tests { }); let mut batch_result = default_vm_batch_result(); - batch_result - .final_execution_state - .deduplicated_storage_log_queries = - storage_logs.iter().map(|query| query.log_query).collect(); + batch_result.final_execution_state.deduplicated_storage_logs = + storage_logs.iter().map(|log| log.log).collect(); batch_result.state_diffs = Some( storage_logs .into_iter() - .filter(|&log| log.log_type == StorageLogQueryType::InitialWrite) - .map(|log| { - let key = StorageKey::new( - AccountTreeId::new(log.log_query.address), - u256_to_h256(log.log_query.key), - ); - StateDiffRecord { - address: log.log_query.address, - key: log.log_query.key, - derived_key: key.hashed_key().0, - enumeration_index: 0, - initial_value: log.log_query.read_value, - final_value: log.log_query.written_value, - } + .filter(|&log| log.log.kind == StorageLogKind::InitialWrite) + .map(|log| StateDiffRecord { + address: *log.log.key.address(), + key: h256_to_u256(*log.log.key.key()), + derived_key: log.log.key.hashed_key().0, + enumeration_index: 0, + initial_value: h256_to_u256(log.previous_value), + final_value: h256_to_u256(log.log.value), }) .collect(), ); diff --git a/core/node/state_keeper/src/io/seal_logic/l2_block_seal_subtasks.rs b/core/node/state_keeper/src/io/seal_logic/l2_block_seal_subtasks.rs index 68fbd62bd973..fabdc855fa47 100644 --- a/core/node/state_keeper/src/io/seal_logic/l2_block_seal_subtasks.rs +++ b/core/node/state_keeper/src/io/seal_logic/l2_block_seal_subtasks.rs @@ -160,17 +160,16 @@ impl L2BlockSealSubtask for InsertStorageLogsSubtask { connection: &mut Connection<'_, Core>, ) -> anyhow::Result<()> { let is_fictive = command.is_l2_block_fictive(); - let write_logs = command.extract_deduplicated_write_logs(is_fictive); + let write_logs = command.extract_deduplicated_write_logs(); let progress = L2_BLOCK_METRICS.start(L2BlockSealStage::InsertStorageLogs, is_fictive); - let write_log_count: usize = write_logs.iter().map(|(_, logs)| logs.len()).sum(); connection .storage_logs_dal() .insert_storage_logs(command.l2_block.number, &write_logs) .await?; - progress.observe(write_log_count); + progress.observe(write_logs.len()); Ok(()) } @@ -377,9 +376,8 @@ mod tests { block::L2BlockHeader, l2_to_l1_log::{L2ToL1Log, UserL2ToL1Log}, tx::{tx_execution_info::TxExecutionStatus, TransactionExecutionResult}, - zk_evm_types::{LogQuery, Timestamp}, - AccountTreeId, Address, L1BatchNumber, ProtocolVersionId, StorageKey, StorageLogQuery, - StorageLogQueryType, VmEvent, U256, + AccountTreeId, Address, L1BatchNumber, ProtocolVersionId, StorageKey, StorageLog, + StorageLogKind, StorageLogWithPreviousValue, VmEvent, }; use zksync_utils::h256_to_u256; @@ -420,21 +418,13 @@ mod tests { }]; let storage_key = StorageKey::new(AccountTreeId::new(Address::zero()), H256::zero()); let storage_value = H256::from_low_u64_be(1); - let storage_logs = vec![StorageLogQuery { - log_query: LogQuery { - timestamp: Timestamp(0), - tx_number_in_block: 0, - aux_byte: 0, - shard_id: 0, - address: *storage_key.address(), - key: h256_to_u256(*storage_key.key()), - read_value: U256::zero(), - written_value: h256_to_u256(storage_value), - rw_flag: true, - rollback: false, - is_service: false, + let storage_logs = vec![StorageLogWithPreviousValue { + log: StorageLog { + key: storage_key, + value: storage_value, + kind: StorageLogKind::InitialWrite, }, - log_type: StorageLogQueryType::InitialWrite, + previous_value: H256::zero(), }]; let user_l2_to_l1_logs = vec![UserL2ToL1Log(L2ToL1Log { shard_id: 0, diff --git a/core/node/state_keeper/src/io/seal_logic/mod.rs b/core/node/state_keeper/src/io/seal_logic/mod.rs index 3e8277485d2f..5aedb85b8131 100644 --- a/core/node/state_keeper/src/io/seal_logic/mod.rs +++ b/core/node/state_keeper/src/io/seal_logic/mod.rs @@ -22,7 +22,6 @@ use zksync_types::{ TransactionExecutionResult, }, utils::display_timestamp, - zk_evm_types::LogQuery, AccountTreeId, Address, ExecuteTransactionCommon, ProtocolVersionId, StorageKey, StorageLog, Transaction, VmEvent, H256, }; @@ -82,7 +81,7 @@ impl UpdatesManager { progress.observe( finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .len(), ); @@ -90,7 +89,7 @@ impl UpdatesManager { let (dedup_writes_count, dedup_reads_count) = log_query_write_read_counts( finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .iter(), ); @@ -173,9 +172,9 @@ impl UpdatesManager { let progress = L1_BATCH_METRICS.start(L1BatchSealStage::InsertProtectiveReads); let protective_reads: Vec<_> = finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .iter() - .filter(|log_query| !log_query.rw_flag) + .filter(|log_query| !log_query.is_write()) .copied() .collect(); transaction @@ -204,18 +203,13 @@ impl UpdatesManager { } else { let deduplicated_writes = finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .iter() - .filter(|log_query| log_query.rw_flag); + .filter(|log_query| log_query.is_write()); let deduplicated_writes_hashed_keys: Vec<_> = deduplicated_writes .clone() - .map(|log| { - H256(StorageKey::raw_hashed_key( - &log.address, - &u256_to_h256(log.key), - )) - }) + .map(|log| log.key.hashed_key()) .collect(); let all_writes_len = deduplicated_writes_hashed_keys.len(); let non_initial_writes = transaction @@ -226,9 +220,7 @@ impl UpdatesManager { ( deduplicated_writes .filter_map(|log| { - let key = - StorageKey::new(AccountTreeId::new(log.address), u256_to_h256(log.key)); - (!non_initial_writes.contains(&key.hashed_key())).then_some(key) + (!non_initial_writes.contains(&log.key.hashed_key())).then_some(log.key) }) .collect(), all_writes_len, @@ -435,55 +427,22 @@ impl L2BlockSealCommand { "event transaction index {tx_index} is outside of the expected range {tx_index_range:?}" ); } - for storage_log in &self.l2_block.storage_logs { - let tx_index = storage_log.log_query.tx_number_in_block as usize; - anyhow::ensure!( - tx_index_range.contains(&tx_index), - "log transaction index {tx_index} is outside of the expected range {tx_index_range:?}" - ); - } Ok(()) } - fn extract_deduplicated_write_logs(&self, is_fictive: bool) -> Vec<(H256, Vec)> { + fn extract_deduplicated_write_logs(&self) -> Vec { let mut storage_writes_deduplicator = StorageWritesDeduplicator::new(); storage_writes_deduplicator.apply( self.l2_block .storage_logs .iter() - .filter(|log| log.log_query.rw_flag), + .filter(|log| log.log.is_write()), ); let deduplicated_logs = storage_writes_deduplicator.into_modified_key_values(); deduplicated_logs .into_iter() - .map( - |( - key, - ModifiedSlot { - value, tx_index, .. - }, - )| (tx_index, (key, value)), - ) - .sorted_by_key(|(tx_index, _)| *tx_index) - .group_by(|(tx_index, _)| *tx_index) - .into_iter() - .map(|(tx_index, logs)| { - let tx_hash = if is_fictive { - assert_eq!(tx_index as usize, self.first_tx_index); - H256::zero() - } else { - self.transaction(tx_index as usize).hash() - }; - ( - tx_hash, - logs.into_iter() - .map(|(_, (key, value))| { - StorageLog::new_write_log(key, u256_to_h256(value)) - }) - .collect(), - ) - }) + .map(|(key, ModifiedSlot { value, .. })| StorageLog::new_write_log(key, value)) .collect() } @@ -601,12 +560,12 @@ fn l1_l2_tx_count(executed_transactions: &[TransactionExecutionResult]) -> (usiz (l1_tx_count, l2_tx_count) } -fn log_query_write_read_counts<'a>(logs: impl Iterator) -> (usize, usize) { +fn log_query_write_read_counts<'a>(logs: impl Iterator) -> (usize, usize) { let mut reads_count = 0; let mut writes_count = 0; for log in logs { - if log.rw_flag { + if log.is_write() { writes_count += 1; } else { reads_count += 1; diff --git a/core/node/state_keeper/src/io/tests/mod.rs b/core/node/state_keeper/src/io/tests/mod.rs index 2587bca237f2..ee0e39ed0618 100644 --- a/core/node/state_keeper/src/io/tests/mod.rs +++ b/core/node/state_keeper/src/io/tests/mod.rs @@ -241,7 +241,7 @@ async fn processing_storage_logs_when_sealing_l2_block() { Query::RepeatedWrite(U256::from(1), U256::from(4)), ), ]; - let execution_result = create_execution_result(0, storage_logs); + let execution_result = create_execution_result(storage_logs); l2_block.extend_from_executed_transaction( tx, execution_result, @@ -259,7 +259,7 @@ async fn processing_storage_logs_when_sealing_l2_block() { Query::RepeatedWrite(U256::from(3), U256::from(6)), ), ]; - let execution_result = create_execution_result(1, storage_logs); + let execution_result = create_execution_result(storage_logs); l2_block.extend_from_executed_transaction( tx, execution_result, @@ -345,9 +345,9 @@ async fn processing_events_when_sealing_l2_block() { }); let events: Vec<_> = events.collect(); - for (i, events_chunk) in events.chunks(4).enumerate() { + for events_chunk in events.chunks(4) { let tx = create_transaction(10, 100); - let mut execution_result = create_execution_result(i as u16, []); + let mut execution_result = create_execution_result([]); execution_result.logs.events = events_chunk.to_vec(); l2_block.extend_from_executed_transaction( tx, @@ -454,7 +454,7 @@ async fn l2_block_processing_after_snapshot_recovery(commitment_mode: L1BatchCom let tx_hash = tx.hash(); updates.extend_from_executed_transaction( tx.into(), - create_execution_result(0, []), + create_execution_result([]), vec![], BlockGasCount::default(), ExecutionMetrics::default(), diff --git a/core/node/state_keeper/src/mempool_actor.rs b/core/node/state_keeper/src/mempool_actor.rs index 9725fc89df3d..85a68069e00b 100644 --- a/core/node/state_keeper/src/mempool_actor.rs +++ b/core/node/state_keeper/src/mempool_actor.rs @@ -192,7 +192,7 @@ mod tests { let nonce_log = StorageLog::new_write_log(nonce_key, u256_to_h256(42.into())); storage .storage_logs_dal() - .insert_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![nonce_log])]) + .insert_storage_logs(L2BlockNumber(0), &[nonce_log]) .await .unwrap(); @@ -352,7 +352,7 @@ mod tests { let mut storage = pool.connection().await.unwrap(); storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![nonce_log])]) + .append_storage_logs(L2BlockNumber(0), &[nonce_log]) .await .unwrap(); storage diff --git a/core/node/state_keeper/src/seal_criteria/mod.rs b/core/node/state_keeper/src/seal_criteria/mod.rs index a721c53b6467..ff231107326a 100644 --- a/core/node/state_keeper/src/seal_criteria/mod.rs +++ b/core/node/state_keeper/src/seal_criteria/mod.rs @@ -286,7 +286,7 @@ mod tests { fn apply_tx_to_manager(tx: Transaction, manager: &mut UpdatesManager) { manager.extend_from_executed_transaction( tx, - create_execution_result(0, []), + create_execution_result([]), vec![], BlockGasCount::default(), ExecutionMetrics::default(), diff --git a/core/node/state_keeper/src/testonly/mod.rs b/core/node/state_keeper/src/testonly/mod.rs index 3ba61949516b..3f7244a2fb75 100644 --- a/core/node/state_keeper/src/testonly/mod.rs +++ b/core/node/state_keeper/src/testonly/mod.rs @@ -19,8 +19,8 @@ use zksync_state::ReadStorageFactory; use zksync_test_account::Account; use zksync_types::{ fee::Fee, utils::storage_key_for_standard_token_balance, AccountTreeId, Address, Execute, - L1BatchNumber, L2BlockNumber, PriorityOpId, StorageLog, Transaction, H256, - L2_BASE_TOKEN_ADDRESS, SYSTEM_CONTEXT_MINIMAL_BASE_FEE, U256, + L1BatchNumber, L2BlockNumber, PriorityOpId, StorageLog, Transaction, L2_BASE_TOKEN_ADDRESS, + SYSTEM_CONTEXT_MINIMAL_BASE_FEE, U256, }; use zksync_utils::u256_to_h256; @@ -44,7 +44,7 @@ pub(super) fn default_vm_batch_result() -> FinishedL1Batch { }, final_execution_state: CurrentExecutionState { events: vec![], - deduplicated_storage_log_queries: vec![], + deduplicated_storage_logs: vec![], used_contract_hashes: vec![], user_l2_to_l1_logs: vec![], system_logs: vec![], @@ -130,7 +130,7 @@ pub async fn fund(pool: &ConnectionPool, addresses: &[Address]) { storage .storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![storage_log])]) + .append_storage_logs(L2BlockNumber(0), &[storage_log]) .await .unwrap(); if storage diff --git a/core/node/state_keeper/src/tests/mod.rs b/core/node/state_keeper/src/tests/mod.rs index b5560605eedf..ee716df2e691 100644 --- a/core/node/state_keeper/src/tests/mod.rs +++ b/core/node/state_keeper/src/tests/mod.rs @@ -21,10 +21,11 @@ use zksync_types::{ block::{BlockGasCount, L2BlockExecutionData, L2BlockHasher}, fee_model::{BatchFeeInput, PubdataIndependentBatchFeeModelInput}, tx::tx_execution_info::ExecutionMetrics, - zk_evm_types::{LogQuery, Timestamp}, - Address, L1BatchNumber, L2BlockNumber, L2ChainId, ProtocolVersionId, StorageLogQuery, - StorageLogQueryType, Transaction, H256, U256, ZKPORTER_IS_AVAILABLE, + AccountTreeId, Address, L1BatchNumber, L2BlockNumber, L2ChainId, ProtocolVersionId, StorageKey, + StorageLog, StorageLogKind, StorageLogWithPreviousValue, Transaction, H256, U256, + ZKPORTER_IS_AVAILABLE, }; +use zksync_utils::u256_to_h256; use crate::{ batch_executor::TxExecutionResult, @@ -112,12 +113,11 @@ pub(super) fn create_transaction(fee_per_gas: u64, gas_per_pubdata: u64) -> Tran } pub(super) fn create_execution_result( - tx_number_in_block: u16, storage_logs: impl IntoIterator, ) -> VmExecutionResultAndLogs { let storage_logs: Vec<_> = storage_logs .into_iter() - .map(|(key, query)| query.into_log(key, tx_number_in_block)) + .map(|(key, query)| query.into_log(key)) .collect(); let total_log_queries = storage_logs.len() + 2; @@ -152,34 +152,24 @@ pub(super) enum Query { } impl Query { - fn into_log(self, key: U256, tx_number_in_block: u16) -> StorageLogQuery { - let log_type = match self { - Self::Read(_) => StorageLogQueryType::Read, - Self::InitialWrite(_) => StorageLogQueryType::InitialWrite, - Self::RepeatedWrite(_, _) => StorageLogQueryType::RepeatedWrite, - }; - - StorageLogQuery { - log_query: LogQuery { - timestamp: Timestamp(0), - tx_number_in_block, - aux_byte: 0, - shard_id: 0, - address: Address::default(), - key, - read_value: match self { - Self::Read(prev) | Self::RepeatedWrite(prev, _) => prev, - Self::InitialWrite(_) => U256::zero(), + fn into_log(self, key: U256) -> StorageLogWithPreviousValue { + StorageLogWithPreviousValue { + log: StorageLog { + kind: match self { + Self::Read(_) => StorageLogKind::Read, + Self::InitialWrite(_) => StorageLogKind::InitialWrite, + Self::RepeatedWrite(_, _) => StorageLogKind::RepeatedWrite, }, - written_value: match self { - Self::Read(_) => U256::zero(), - Self::InitialWrite(value) | Self::RepeatedWrite(_, value) => value, - }, - rw_flag: !matches!(self, Self::Read(_)), - rollback: false, - is_service: false, + key: StorageKey::new(AccountTreeId::new(Address::default()), u256_to_h256(key)), + value: u256_to_h256(match self { + Query::Read(_) => U256::zero(), + Query::InitialWrite(value) | Query::RepeatedWrite(_, value) => value, + }), }, - log_type, + previous_value: u256_to_h256(match self { + Query::Read(value) | Query::RepeatedWrite(value, _) => value, + Query::InitialWrite(_) => U256::zero(), + }), } } } diff --git a/core/node/state_keeper/src/updates/l1_batch_updates.rs b/core/node/state_keeper/src/updates/l1_batch_updates.rs index 6becfae2b7ac..0670b06db7d7 100644 --- a/core/node/state_keeper/src/updates/l1_batch_updates.rs +++ b/core/node/state_keeper/src/updates/l1_batch_updates.rs @@ -74,7 +74,7 @@ mod tests { l2_block_accumulator.extend_from_executed_transaction( tx, - create_execution_result(0, []), + create_execution_result([]), BlockGasCount::default(), ExecutionMetrics::default(), vec![], diff --git a/core/node/state_keeper/src/updates/l2_block_updates.rs b/core/node/state_keeper/src/updates/l2_block_updates.rs index 34cfad44f934..93e0a481ebc3 100644 --- a/core/node/state_keeper/src/updates/l2_block_updates.rs +++ b/core/node/state_keeper/src/updates/l2_block_updates.rs @@ -10,7 +10,7 @@ use zksync_types::{ l2_to_l1_log::{SystemL2ToL1Log, UserL2ToL1Log}, tx::{tx_execution_info::TxExecutionStatus, ExecutionMetrics, TransactionExecutionResult}, vm_trace::Call, - L2BlockNumber, ProtocolVersionId, StorageLogQuery, Transaction, VmEvent, H256, + L2BlockNumber, ProtocolVersionId, StorageLogWithPreviousValue, Transaction, VmEvent, H256, }; use zksync_utils::bytecode::{hash_bytecode, CompressedBytecodeInfo}; @@ -20,7 +20,7 @@ use crate::metrics::KEEPER_METRICS; pub struct L2BlockUpdates { pub executed_transactions: Vec, pub events: Vec, - pub storage_logs: Vec, + pub storage_logs: Vec, pub user_l2_to_l1_logs: Vec, pub system_l2_to_l1_logs: Vec, pub new_factory_deps: HashMap>, @@ -202,7 +202,7 @@ mod tests { accumulator.extend_from_executed_transaction( tx, - create_execution_result(0, []), + create_execution_result([]), BlockGasCount::default(), ExecutionMetrics::default(), vec![], diff --git a/core/node/state_keeper/src/updates/mod.rs b/core/node/state_keeper/src/updates/mod.rs index 6f920464cc06..c78607147468 100644 --- a/core/node/state_keeper/src/updates/mod.rs +++ b/core/node/state_keeper/src/updates/mod.rs @@ -221,7 +221,7 @@ mod tests { let tx = create_transaction(10, 100); updates_manager.extend_from_executed_transaction( tx, - create_execution_result(0, []), + create_execution_result([]), vec![], new_block_gas_count(), ExecutionMetrics::default(), diff --git a/core/node/test_utils/src/lib.rs b/core/node/test_utils/src/lib.rs index d0dfe367c21d..a77e0aea2c0c 100644 --- a/core/node/test_utils/src/lib.rs +++ b/core/node/test_utils/src/lib.rs @@ -317,10 +317,7 @@ pub async fn recover( .unwrap(); storage .storage_logs_dal() - .insert_storage_logs( - snapshot.l2_block.number, - &[(H256::zero(), snapshot.storage_logs)], - ) + .insert_storage_logs(snapshot.l2_block.number, &snapshot.storage_logs) .await .unwrap(); diff --git a/core/node/vm_runner/src/impls/protective_reads.rs b/core/node/vm_runner/src/impls/protective_reads.rs index 8fcb5c6b3f08..6a8d85e3bd49 100644 --- a/core/node/vm_runner/src/impls/protective_reads.rs +++ b/core/node/vm_runner/src/impls/protective_reads.rs @@ -5,8 +5,7 @@ use async_trait::async_trait; use tokio::sync::watch; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal}; use zksync_state_keeper::{MainBatchExecutor, StateKeeperOutputHandler, UpdatesManager}; -use zksync_types::{zk_evm_types::LogQuery, AccountTreeId, L1BatchNumber, L2ChainId, StorageKey}; -use zksync_utils::u256_to_h256; +use zksync_types::{L1BatchNumber, L2ChainId, StorageLog}; use crate::{ storage::StorageSyncTask, ConcurrentOutputHandlerFactory, ConcurrentOutputHandlerFactoryTask, @@ -140,11 +139,11 @@ impl StateKeeperOutputHandler for ProtectiveReadsOutputHandler { .finished .as_ref() .context("L1 batch is not actually finished")?; - let (_, protective_reads): (Vec, Vec) = finished_batch + let (_, protective_reads): (Vec, Vec) = finished_batch .final_execution_state - .deduplicated_storage_log_queries + .deduplicated_storage_logs .iter() - .partition(|log_query| log_query.rw_flag); + .partition(|log_query| log_query.is_write()); let mut connection = self .pool @@ -156,12 +155,12 @@ impl StateKeeperOutputHandler for ProtectiveReadsOutputHandler { .await?; for protective_read in protective_reads { - let address = AccountTreeId::new(protective_read.address); - let key = u256_to_h256(protective_read.key); - if !expected_protective_reads.remove(&StorageKey::new(address, key)) { + let address = protective_read.key.address(); + let key = protective_read.key.key(); + if !expected_protective_reads.remove(&protective_read.key) { tracing::error!( l1_batch_number = %updates_manager.l1_batch.number, - address = %protective_read.address, + address = %address, key = %key, "VM runner produced a protective read that did not happen in state keeper" ); diff --git a/core/node/vm_runner/src/tests/mod.rs b/core/node/vm_runner/src/tests/mod.rs index 0d106235f713..52c4db4bb486 100644 --- a/core/node/vm_runner/src/tests/mod.rs +++ b/core/node/vm_runner/src/tests/mod.rs @@ -235,7 +235,7 @@ async fn store_l1_batches( let value = StorageValue::random(); written_keys.push(key); logs.push(StorageLog { - kind: StorageLogKind::Write, + kind: StorageLogKind::RepeatedWrite, key, value, }); @@ -245,7 +245,7 @@ async fn store_l1_batches( factory_deps.insert(H256::random(), rng.gen::<[u8; 32]>().into()); } conn.storage_logs_dal() - .insert_storage_logs(l2_block_number, &[(tx.hash(), logs)]) + .insert_storage_logs(l2_block_number, &logs) .await?; conn.storage_logs_dedup_dal() .insert_initial_writes(l1_batch_number, &written_keys) @@ -343,7 +343,7 @@ async fn fund(pool: &ConnectionPool, accounts: &[Account]) { let storage_log = StorageLog::new_write_log(key, value); conn.storage_logs_dal() - .append_storage_logs(L2BlockNumber(0), &[(H256::zero(), vec![storage_log])]) + .append_storage_logs(L2BlockNumber(0), &[storage_log]) .await .unwrap(); if conn From 7f4e6ac28f99ab7394131d44a8e7243b8cbe3727 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Wed, 19 Jun 2024 18:39:50 +0300 Subject: [PATCH 28/43] refactor(db): Combine storage log pruning into single query (#2279) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Minor follow-up for https://github.com/matter-labs/zksync-era/pull/2268 that combines both parts of log pruning into a single query. ## Why ❔ Easier to maintain and could be slightly more efficient since intermediate data doesn't need to travel from Postgres to the node and back. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- ...60cd2f3d5223add676591cb0577e0a77403cb.json | 16 --- ...9b5c09854efaa4c0a35466b138587dce03f25.json | 15 +++ ...94d8d631d56c5753f4e944f1cdf3e05b04a8c.json | 35 ------- core/lib/dal/src/pruning_dal/mod.rs | 99 ++++++------------- core/node/db_pruner/src/metrics.rs | 5 +- 5 files changed, 47 insertions(+), 123 deletions(-) delete mode 100644 core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json create mode 100644 core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json delete mode 100644 core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json diff --git a/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json b/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json deleted file mode 100644 index 7ecce5be1f35..000000000000 --- a/core/lib/dal/.sqlx/query-327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n DELETE FROM storage_logs USING UNNEST($1::bytea[], $2::BIGINT[], $3::INT[]) AS new_logs (hashed_key, miniblock_number, operation_number)\n WHERE\n storage_logs.hashed_key = new_logs.hashed_key\n AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number)\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "ByteaArray", - "Int8Array", - "Int4Array" - ] - }, - "nullable": [] - }, - "hash": "327974ef6d0c7edf56339d310ec60cd2f3d5223add676591cb0577e0a77403cb" -} diff --git a/core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json b/core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json new file mode 100644 index 000000000000..93d1966f370c --- /dev/null +++ b/core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n WITH\n new_logs AS MATERIALIZED (\n SELECT DISTINCT\n ON (hashed_key) hashed_key,\n miniblock_number,\n operation_number\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n ORDER BY\n hashed_key,\n miniblock_number DESC,\n operation_number DESC\n )\n DELETE FROM storage_logs USING new_logs\n WHERE\n storage_logs.hashed_key = new_logs.hashed_key\n AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25" +} diff --git a/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json b/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json deleted file mode 100644 index ffb51e0dd865..000000000000 --- a/core/lib/dal/.sqlx/query-8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT DISTINCT\n ON (hashed_key) hashed_key,\n miniblock_number,\n operation_number\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n ORDER BY\n hashed_key,\n miniblock_number DESC,\n operation_number DESC\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "hashed_key", - "type_info": "Bytea" - }, - { - "ordinal": 1, - "name": "miniblock_number", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "operation_number", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [ - false, - false, - false - ] - }, - "hash": "8c2f1f7bccc6af93714a74f732f94d8d631d56c5753f4e944f1cdf3e05b04a8c" -} diff --git a/core/lib/dal/src/pruning_dal/mod.rs b/core/lib/dal/src/pruning_dal/mod.rs index 16f85f2e0fad..7f30af034e2b 100644 --- a/core/lib/dal/src/pruning_dal/mod.rs +++ b/core/lib/dal/src/pruning_dal/mod.rs @@ -1,6 +1,5 @@ use std::ops; -use itertools::Itertools; use zksync_db_connection::{connection::Connection, error::DalResult, instrument::InstrumentExt}; use zksync_types::{L1BatchNumber, L2BlockNumber}; @@ -28,7 +27,6 @@ pub struct PruningInfo { pub struct HardPruningStats { pub deleted_l1_batches: u64, pub deleted_l2_blocks: u64, - pub overwriting_logs: u64, pub deleted_storage_logs: u64, pub deleted_events: u64, pub deleted_call_traces: u64, @@ -42,14 +40,6 @@ enum PruneType { Hard, } -/// Raw database presentation of a primary key in the `miniblocks` table. -#[derive(Debug)] -struct StorageLogPrimaryKey { - hashed_key: Vec, - miniblock_number: i64, - operation_number: i32, -} - impl PruningDal<'_, '_> { pub async fn get_pruning_info(&mut self) -> DalResult { let pruning_info = sqlx::query!( @@ -183,18 +173,9 @@ impl PruningDal<'_, '_> { self.clear_transaction_fields(first_l2_block_to_prune..=last_l2_block_to_prune) .await?; - // Storage log pruning is designed to use deterministic indexes and thus have predictable performance. - // - // - `get_pks_for_latest_logs` is guaranteed to use the block number index (that's the only WHERE condition), - // and the supplied range of blocks should be reasonably small. - // - `prune_storage_logs` is virtually guaranteed to use the primary key index since the query removes ranges w.r.t. this index. - // - // Combining these two queries or using more sophisticated queries leads to fluctuating performance due to - // unpredictable indexes being used. - let new_logs = self - .get_pks_for_latest_logs(first_l2_block_to_prune..=last_l2_block_to_prune) + let deleted_storage_logs = self + .prune_storage_logs(first_l2_block_to_prune..=last_l2_block_to_prune) .await?; - let deleted_storage_logs = self.prune_storage_logs(&new_logs).await?; let deleted_l1_batches = self.delete_l1_batches(last_l1_batch_to_prune).await?; let deleted_l2_blocks = self.delete_l2_blocks(last_l2_block_to_prune).await?; @@ -204,7 +185,6 @@ impl PruningDal<'_, '_> { deleted_events, deleted_l2_to_l1_logs, deleted_call_traces, - overwriting_logs: new_logs.len() as u64, deleted_storage_logs, } } else { @@ -324,62 +304,45 @@ impl PruningDal<'_, '_> { Ok(execution_result.rows_affected()) } - /// Gets primary keys for all latest logs in the specified L2 block range. - async fn get_pks_for_latest_logs( + /// Removes storage logs overwritten by the specified new logs. + async fn prune_storage_logs( &mut self, l2_blocks_to_prune: ops::RangeInclusive, - ) -> DalResult> { - sqlx::query_as!( - StorageLogPrimaryKey, - r#" - SELECT DISTINCT - ON (hashed_key) hashed_key, - miniblock_number, - operation_number - FROM - storage_logs - WHERE - miniblock_number BETWEEN $1 AND $2 - ORDER BY - hashed_key, - miniblock_number DESC, - operation_number DESC - "#, - i64::from(l2_blocks_to_prune.start().0), - i64::from(l2_blocks_to_prune.end().0) - ) - .instrument("hard_prune_batches_range#get_latest_logs") - .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) - .report_latency() - .fetch_all(self.storage) - .await - } - - /// Removes storage logs overwritten by the specified new logs. - async fn prune_storage_logs(&mut self, new_logs: &[StorageLogPrimaryKey]) -> DalResult { - let (hashed_keys, block_numbers, operation_numbers): (Vec<_>, Vec<_>, Vec<_>) = new_logs - .iter() - .map(|log| { - ( - log.hashed_key.as_slice(), - log.miniblock_number, - log.operation_number, - ) - }) - .multiunzip(); + ) -> DalResult { + // Storage log pruning is designed to use deterministic indexes and thus have predictable performance. + // + // - The WITH query is guaranteed to use the block number index (that's the only WHERE condition), + // and the supplied range of blocks should be reasonably small. + // - The main DELETE query is virtually guaranteed to use the primary key index since it removes ranges w.r.t. this index. + // + // Using more sophisticated queries leads to fluctuating performance due to unpredictable indexes being used. let execution_result = sqlx::query!( r#" - DELETE FROM storage_logs USING UNNEST($1::bytea[], $2::BIGINT[], $3::INT[]) AS new_logs (hashed_key, miniblock_number, operation_number) + WITH + new_logs AS MATERIALIZED ( + SELECT DISTINCT + ON (hashed_key) hashed_key, + miniblock_number, + operation_number + FROM + storage_logs + WHERE + miniblock_number BETWEEN $1 AND $2 + ORDER BY + hashed_key, + miniblock_number DESC, + operation_number DESC + ) + DELETE FROM storage_logs USING new_logs WHERE storage_logs.hashed_key = new_logs.hashed_key AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number) "#, - &hashed_keys as &[&[u8]], - &block_numbers, - &operation_numbers + i64::from(l2_blocks_to_prune.start().0), + i64::from(l2_blocks_to_prune.end().0) ) .instrument("hard_prune_batches_range#prune_storage_logs") - .with_arg("new_logs.len", &new_logs.len()) + .with_arg("l2_blocks_to_prune", &l2_blocks_to_prune) .report_latency() .execute(self.storage) .await?; diff --git a/core/node/db_pruner/src/metrics.rs b/core/node/db_pruner/src/metrics.rs index 1070ad842703..0d4d88513dbc 100644 --- a/core/node/db_pruner/src/metrics.rs +++ b/core/node/db_pruner/src/metrics.rs @@ -16,7 +16,6 @@ enum PrunedEntityType { L1Batch, L2Block, StorageLog, - OverwritingLog, // not really removed; just used to measure query complexity Event, L2ToL1Log, CallTrace, @@ -44,7 +43,6 @@ impl DbPrunerMetrics { let HardPruningStats { deleted_l1_batches, deleted_l2_blocks, - overwriting_logs, deleted_storage_logs, deleted_events, deleted_call_traces, @@ -52,13 +50,12 @@ impl DbPrunerMetrics { } = stats; tracing::info!( "Performed pruning of database, deleted {deleted_l1_batches} L1 batches, {deleted_l2_blocks} L2 blocks, \ - {deleted_storage_logs} storage logs ({overwriting_logs} overwriting logs), \ + {deleted_storage_logs} storage logs, \ {deleted_events} events, {deleted_call_traces} call traces, {deleted_l2_to_l1_logs} L2-to-L1 logs" ); self.deleted_entities[&PrunedEntityType::L1Batch].observe(deleted_l1_batches); self.deleted_entities[&PrunedEntityType::L2Block].observe(deleted_l2_blocks); - self.deleted_entities[&PrunedEntityType::OverwritingLog].observe(overwriting_logs); self.deleted_entities[&PrunedEntityType::StorageLog].observe(deleted_storage_logs); self.deleted_entities[&PrunedEntityType::Event].observe(deleted_events); self.deleted_entities[&PrunedEntityType::L2ToL1Log].observe(deleted_l2_to_l1_logs); From f7f5447cb1d9a74978e0d5b6d752f84d627a30ec Mon Sep 17 00:00:00 2001 From: Yury Akudovich Date: Wed, 19 Jun 2024 17:53:56 +0200 Subject: [PATCH 29/43] ci: Add ci-for-common as required for check to succeed (#2281) --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 881af2367d31..9e4d093e317a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -157,7 +157,7 @@ jobs: name: Github Status Check runs-on: ubuntu-latest if: always() && !cancelled() - needs: [ci-for-core-lint, ci-for-core, ci-for-prover, ci-for-docs, build-core-images, build-contract-verifier, build-prover-images] + needs: [ci-for-core-lint, ci-for-common, ci-for-core, ci-for-prover, ci-for-docs, build-core-images, build-contract-verifier, build-prover-images] steps: - name: Status run: | From f851615ab3753bb9353fd4456a6e49d55d67c626 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Thu, 20 Jun 2024 10:28:10 +0200 Subject: [PATCH 30/43] feat(prover): Add file based config for prover fri (#2184) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Add file based config for prover fri ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- prover/Cargo.lock | 2 ++ prover/prover_fri/Cargo.toml | 2 ++ prover/prover_fri/src/main.rs | 54 +++++++++++++++++++++++------------ 3 files changed, 39 insertions(+), 19 deletions(-) diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 096e3998d0a4..6d6b967fbbe7 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -9291,6 +9291,7 @@ dependencies = [ "anyhow", "async-trait", "circuit_definitions 1.5.0", + "clap 4.5.4", "ctrlc", "futures 0.3.30", "local-ip-address", @@ -9309,6 +9310,7 @@ dependencies = [ "zksync_config", "zksync_env_config", "zksync_object_store", + "zksync_prover_config", "zksync_prover_fri_types", "zksync_prover_fri_utils", "zksync_queued_job_processor", diff --git a/prover/prover_fri/Cargo.toml b/prover/prover_fri/Cargo.toml index 5b618c928ed4..9bce1f2581b8 100644 --- a/prover/prover_fri/Cargo.toml +++ b/prover/prover_fri/Cargo.toml @@ -20,6 +20,7 @@ vlog.workspace = true zksync_object_store.workspace = true zksync_queued_job_processor.workspace = true zksync_prover_fri_utils.workspace = true +zksync_prover_config.workspace = true zksync_prover_fri_types.workspace = true zksync_utils.workspace = true vk_setup_data_generator_server_fri.workspace = true @@ -41,6 +42,7 @@ async-trait.workspace = true local-ip-address.workspace = true reqwest = { workspace = true, features = ["blocking"] } regex.workspace = true +clap = { workspace = true, features = ["derive"] } [features] default = [] diff --git a/prover/prover_fri/src/main.rs b/prover/prover_fri/src/main.rs index 86fd114fa12e..ab2a4d1575c5 100644 --- a/prover/prover_fri/src/main.rs +++ b/prover/prover_fri/src/main.rs @@ -3,6 +3,7 @@ use std::{future::Future, sync::Arc, time::Duration}; use anyhow::Context as _; +use clap::Parser; use local_ip_address::local_ip; use prometheus_exporter::PrometheusExporterConfig; use prover_dal::{ConnectionPool, Prover, ProverDal}; @@ -10,14 +11,10 @@ use tokio::{ sync::{oneshot, watch::Receiver, Notify}, task::JoinHandle, }; -use zksync_config::configs::{ - fri_prover_group::FriProverGroupConfig, DatabaseSecrets, FriProverConfig, ObservabilityConfig, -}; -use zksync_env_config::{ - object_store::{ProverObjectStoreConfig, PublicObjectStoreConfig}, - FromEnv, -}; +use zksync_config::configs::{DatabaseSecrets, FriProverConfig}; +use zksync_env_config::FromEnv; use zksync_object_store::{ObjectStore, ObjectStoreFactory}; +use zksync_prover_config::{load_database_secrets, load_general_config}; use zksync_prover_fri_types::PROVER_PROTOCOL_SEMANTIC_VERSION; use zksync_prover_fri_utils::{get_all_circuit_id_round_tuples_for, region_fetcher::get_zone}; use zksync_queued_job_processor::JobProcessor; @@ -58,8 +55,14 @@ async fn graceful_shutdown(port: u16) -> anyhow::Result #[tokio::main] async fn main() -> anyhow::Result<()> { - let observability_config = - ObservabilityConfig::from_env().context("ObservabilityConfig::from_env()")?; + let opt = Cli::parse(); + + let general_config = load_general_config(opt.config_path).context("general config")?; + let database_secrets = load_database_secrets(opt.secrets_path).context("database secrets")?; + + let observability_config = general_config + .observability + .context("observability config")?; let log_format: vlog::LogFormat = observability_config .log_format .parse() @@ -91,7 +94,7 @@ async fn main() -> anyhow::Result<()> { tracing::info!("No sentry URL was provided"); } - let prover_config = FriProverConfig::from_env().context("FriProverConfig::from_env()")?; + let prover_config = general_config.prover_config.context("fri_prover config")?; let exporter_config = PrometheusExporterConfig::pull(prover_config.prometheus_port); let (stop_signal_sender, stop_signal_receiver) = oneshot::channel(); @@ -104,23 +107,28 @@ async fn main() -> anyhow::Result<()> { .context("Error setting Ctrl+C handler")?; let (stop_sender, stop_receiver) = tokio::sync::watch::channel(false); - let object_store_config = - ProverObjectStoreConfig::from_env().context("ProverObjectStoreConfig::from_env()")?; - let object_store_factory = ObjectStoreFactory::new(object_store_config.0); - let public_object_store_config = - PublicObjectStoreConfig::from_env().context("PublicObjectStoreConfig::from_env()")?; + let prover_object_store_config = prover_config + .prover_object_store + .clone() + .context("prover object store config")?; + let object_store_factory = ObjectStoreFactory::new(prover_object_store_config); + let public_object_store_config = prover_config + .public_object_store + .clone() + .context("public object store config")?; let public_blob_store = match prover_config.shall_save_to_public_bucket { false => None, true => Some( - ObjectStoreFactory::new(public_object_store_config.0) + ObjectStoreFactory::new(public_object_store_config) .create_store() .await?, ), }; let specialized_group_id = prover_config.specialized_group_id; - let circuit_ids_for_round_to_be_proven = FriProverGroupConfig::from_env() - .context("FriProverGroupConfig::from_env()")? + let circuit_ids_for_round_to_be_proven = general_config + .prover_group_config + .context("prover group config")? .get_circuit_ids_for_group_id(specialized_group_id) .unwrap_or_default(); let circuit_ids_for_round_to_be_proven = @@ -131,7 +139,6 @@ async fn main() -> anyhow::Result<()> { specialized_group_id, circuit_ids_for_round_to_be_proven.clone() ); - let database_secrets = DatabaseSecrets::from_env().context("DatabaseSecrets")?; // There are 2 threads using the connection pool: // 1. The prover thread, which is used to update the prover job status. @@ -302,3 +309,12 @@ async fn get_prover_tasks( Ok(tasks) } + +#[derive(Debug, Parser)] +#[command(author = "Matter Labs", version)] +pub(crate) struct Cli { + #[arg(long)] + pub(crate) config_path: Option, + #[arg(long)] + pub(crate) secrets_path: Option, +} From 0a388911914bfcf58785e394db9d5ddce3afdef0 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Thu, 20 Jun 2024 13:15:42 +0400 Subject: [PATCH 31/43] feat: Remove initialize_components function (#2284) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Makes node framework the only way to run main node. Removes `initialize_components` function. ⚠️ Some leftovers are left in `core_leftovers` -- these are still used in some places. Removing them will be tacked separately. ## Why ❔ Part of the migration. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. - [ ] Spellcheck has been run via `zk spellcheck`. --- .github/workflows/ci-core-reusable.yml | 2 +- .github/workflows/ci-zk-toolbox-reusable.yml | 2 +- Cargo.lock | 183 --- core/bin/zksync_server/src/main.rs | 144 +- core/lib/zksync_core_leftovers/Cargo.toml | 100 +- core/lib/zksync_core_leftovers/src/lib.rs | 1276 +---------------- .../src/temp_config_store/mod.rs | 10 +- infrastructure/zk/src/server.ts | 4 - prover/Cargo.lock | 1139 +-------------- 9 files changed, 96 insertions(+), 2764 deletions(-) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index b15bc0c41997..d860d79e06ac 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -205,7 +205,7 @@ jobs: # `sleep 5` because we need to wait until server started properly - name: Run server run: | - ci_run zk server --use-node-framework --components=$SERVER_COMPONENTS &>server.log & + ci_run zk server --components=$SERVER_COMPONENTS &>server.log & ci_run sleep 5 - name: Run contract verifier diff --git a/.github/workflows/ci-zk-toolbox-reusable.yml b/.github/workflows/ci-zk-toolbox-reusable.yml index f3238566eeec..66e54bfa98a4 100644 --- a/.github/workflows/ci-zk-toolbox-reusable.yml +++ b/.github/workflows/ci-zk-toolbox-reusable.yml @@ -90,7 +90,7 @@ jobs: - name: Run server run: | - ci_run zk_inception server --ignore-prerequisites -a --use-node-framework --verbose &>server.log & + ci_run zk_inception server --ignore-prerequisites -a --verbose &>server.log & ci_run sleep 5 - name: Run integration tests diff --git a/Cargo.lock b/Cargo.lock index 1be8739e881d..b013517e0cc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,21 +96,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "allocator-api2" version = "0.2.16" @@ -255,22 +240,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" -[[package]] -name = "async-compression" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f658e2baef915ba0f26f1f7c42bfb8e12f532a01f449a090ded75ae7a07e9ba2" -dependencies = [ - "brotli", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "zstd", - "zstd-safe", -] - [[package]] name = "async-lock" version = "3.2.0" @@ -743,27 +712,6 @@ dependencies = [ "syn_derive", ] -[[package]] -name = "brotli" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516074a47ef4bce09577a3b379392300159ce5b1ba2e501ff1c819950066100f" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da74e2b81409b1b743f8f0c62cc6254afefb8b8e50bbfe3735550f7aeefa3448" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.14.0" @@ -2619,16 +2567,6 @@ dependencies = [ "hashbrown 0.14.2", ] -[[package]] -name = "hdrhistogram" -version = "7.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f19b9f54f7c7f55e31401bb647626ce0cf0f67b0004982ce815b3ee72a02aa8" -dependencies = [ - "byteorder", - "num-traits", -] - [[package]] name = "heck" version = "0.3.3" @@ -2972,16 +2910,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21859b667d66a4c1dacd9df0863b3efb65785474255face87f5bca39dd8407c0" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "itertools" version = "0.10.5" @@ -6849,7 +6777,6 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "hdrhistogram", "indexmap 1.9.3", "pin-project", "pin-project-lite", @@ -6868,8 +6795,6 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" dependencies = [ - "async-compression", - "base64 0.21.5", "bitflags 2.4.1", "bytes", "futures-core", @@ -6877,19 +6802,10 @@ dependencies = [ "http", "http-body", "http-range-header", - "httpdate", - "iri-string", - "mime", - "mime_guess", - "percent-encoding", "pin-project-lite", "tokio", - "tokio-util", - "tower", "tower-layer", "tower-service", - "tracing", - "uuid", ] [[package]] @@ -7182,7 +7098,6 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ - "getrandom", "serde", ] @@ -8335,93 +8250,13 @@ name = "zksync_core_leftovers" version = "0.1.0" dependencies = [ "anyhow", - "assert_matches", - "async-trait", - "axum", - "backon", - "chrono", "ctrlc", - "dashmap", - "futures 0.3.28", - "governor", - "hex", - "itertools 0.10.5", - "jsonrpsee", - "lru", - "multivm", - "once_cell", - "pin-project-lite", - "prometheus_exporter", - "prost 0.12.1", - "prover_dal", - "rand 0.8.5", - "reqwest", - "secrecy", - "serde", - "serde_json", "serde_yaml", - "tempfile", - "test-casing", - "test-log", - "thiserror", - "thread_local", "tokio", - "tower", - "tower-http", - "tracing", - "vise", - "vlog", - "vm_utils", - "zksync_circuit_breaker", - "zksync_commitment_generator", - "zksync_concurrency", "zksync_config", - "zksync_consensus_bft", - "zksync_consensus_crypto", - "zksync_consensus_executor", - "zksync_consensus_network", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_contract_verification_server", - "zksync_contracts", "zksync_dal", - "zksync_db_connection", - "zksync_eth_client", - "zksync_eth_sender", - "zksync_eth_signer", - "zksync_eth_watch", - "zksync_health_check", - "zksync_house_keeper", - "zksync_l1_contract_interface", - "zksync_mempool", - "zksync_merkle_tree", - "zksync_metadata_calculator", - "zksync_mini_merkle_tree", - "zksync_node_api_server", - "zksync_node_consensus", - "zksync_node_fee_model", "zksync_node_genesis", - "zksync_node_sync", - "zksync_node_test_utils", - "zksync_object_store", - "zksync_proof_data_handler", "zksync_protobuf", - "zksync_protobuf_build", - "zksync_protobuf_config", - "zksync_prover_interface", - "zksync_queued_job_processor", - "zksync_shared_metrics", - "zksync_state", - "zksync_state_keeper", - "zksync_storage", - "zksync_system_constants", - "zksync_tee_verifier", - "zksync_tee_verifier_input_producer", - "zksync_test_account", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", ] [[package]] @@ -9500,24 +9335,6 @@ dependencies = [ "zksync_types", ] -[[package]] -name = "zstd" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bffb3309596d527cfcba7dfc6ed6052f1d39dfbd7c867aa2e865e4a449c10110" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43747c7422e2924c11144d5229878b98180ef8b06cca4ab5af37afc8a8d8ea3e" -dependencies = [ - "zstd-sys", -] - [[package]] name = "zstd-sys" version = "2.0.9+zstd.1.5.5" diff --git a/core/bin/zksync_server/src/main.rs b/core/bin/zksync_server/src/main.rs index 1c54895863c0..dfb11b55da92 100644 --- a/core/bin/zksync_server/src/main.rs +++ b/core/bin/zksync_server/src/main.rs @@ -1,4 +1,4 @@ -use std::{str::FromStr, time::Duration}; +use std::str::FromStr; use anyhow::Context as _; use clap::Parser; @@ -20,14 +20,12 @@ use zksync_config::{ GenesisConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, }; use zksync_core_leftovers::{ - genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler, + genesis_init, is_genesis_needed, temp_config_store::{decode_yaml_repr, TempConfigStore}, Component, Components, }; use zksync_env_config::FromEnv; use zksync_eth_client::clients::Client; -use zksync_storage::RocksDB; -use zksync_utils::wait_for_tasks::ManagedTasks; use crate::node_builder::MainNodeBuilder; @@ -67,7 +65,8 @@ struct Cli { /// Path to the yaml with genesis. If set, it will be used instead of env vars. #[arg(long)] genesis_path: Option, - /// Run the node using the node framework. + /// Used to enable node framework. + /// Now the node framework is used by default and this argument is left for backward compatibility. #[arg(long)] use_node_framework: bool, } @@ -88,8 +87,7 @@ impl FromStr for ComponentsToRun { } } -#[tokio::main] -async fn main() -> anyhow::Result<()> { +fn main() -> anyhow::Result<()> { let opt = Cli::parse(); // Load env config and use it if file config is not provided @@ -181,32 +179,10 @@ async fn main() -> anyhow::Result<()> { } }; - let database_secrets = secrets.database.clone().context("DatabaseSecrets")?; - - if opt.genesis || is_genesis_needed(&database_secrets).await { - genesis_init(genesis.clone(), &database_secrets) - .await - .context("genesis_init")?; - - if let Some(ecosystem_contracts) = &contracts_config.ecosystem_contracts { - let l1_secrets = secrets.l1.as_ref().context("l1_screts")?; - let query_client = Client::http(l1_secrets.l1_rpc_url.clone()) - .context("Ethereum client")? - .for_network(genesis.l1_chain_id.into()) - .build(); - zksync_node_genesis::save_set_chain_id_tx( - &query_client, - contracts_config.diamond_proxy_addr, - ecosystem_contracts.state_transition_proxy_addr, - &database_secrets, - ) - .await - .context("Failed to save SetChainId upgrade transaction")?; - } - - if opt.genesis { - return Ok(()); - } + run_genesis_if_needed(opt.genesis, &genesis, &contracts_config, &secrets)?; + if opt.genesis { + // If genesis is requested, we don't need to run the node. + return Ok(()); } let components = if opt.rebuild_tree { @@ -215,69 +191,55 @@ async fn main() -> anyhow::Result<()> { opt.components.0 }; - // If the node framework is used, run the node. - if opt.use_node_framework { - // We run the node from a different thread, since the current thread is in tokio context. - std::thread::spawn(move || -> anyhow::Result<()> { - let node = MainNodeBuilder::new( - configs, - wallets, - genesis, - contracts_config, - secrets, - consensus, - ) - .build(components)?; - node.run()?; - Ok(()) - }) - .join() - .expect("Failed to run the node")?; - - return Ok(()); - } - - // Run core actors. - let sigint_receiver = setup_sigint_handler(); - let (core_task_handles, stop_sender, health_check_handle) = initialize_components( - &configs, - &wallets, - &genesis, - &contracts_config, - &components, - &secrets, + let node = MainNodeBuilder::new( + configs, + wallets, + genesis, + contracts_config, + secrets, consensus, ) - .await - .context("Unable to start Core actors")?; - - tracing::info!("Running {} core task handlers", core_task_handles.len()); - - let mut tasks = ManagedTasks::new(core_task_handles); - tokio::select! { - _ = tasks.wait_single() => {}, - _ = sigint_receiver => { - tracing::info!("Stop signal received, shutting down"); - }, - } - - stop_sender.send(true).ok(); - tokio::task::spawn_blocking(RocksDB::await_rocksdb_termination) - .await - .context("error waiting for RocksDB instances to drop")?; - let complete_timeout = - if components.contains(&Component::HttpApi) || components.contains(&Component::WsApi) { - // Increase timeout because of complicated graceful shutdown procedure for API servers. - Duration::from_secs(30) - } else { - Duration::from_secs(5) - }; - tasks.complete(complete_timeout).await; - health_check_handle.stop().await; - tracing::info!("Stopped"); + .build(components)?; + node.run()?; Ok(()) } +fn run_genesis_if_needed( + force_genesis: bool, + genesis: &GenesisConfig, + contracts_config: &ContractsConfig, + secrets: &Secrets, +) -> anyhow::Result<()> { + let tokio_runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + tokio_runtime.block_on(async move { + let database_secrets = secrets.database.clone().context("DatabaseSecrets")?; + if force_genesis || is_genesis_needed(&database_secrets).await { + genesis_init(genesis.clone(), &database_secrets) + .await + .context("genesis_init")?; + + if let Some(ecosystem_contracts) = &contracts_config.ecosystem_contracts { + let l1_secrets = secrets.l1.as_ref().context("l1_screts")?; + let query_client = Client::http(l1_secrets.l1_rpc_url.clone()) + .context("Ethereum client")? + .for_network(genesis.l1_chain_id.into()) + .build(); + zksync_node_genesis::save_set_chain_id_tx( + &query_client, + contracts_config.diamond_proxy_addr, + ecosystem_contracts.state_transition_proxy_addr, + &database_secrets, + ) + .await + .context("Failed to save SetChainId upgrade transaction")?; + } + } + Ok(()) + }) +} + fn load_env_config() -> anyhow::Result { Ok(TempConfigStore { postgres_config: PostgresConfig::from_env().ok(), diff --git a/core/lib/zksync_core_leftovers/Cargo.toml b/core/lib/zksync_core_leftovers/Cargo.toml index c394342c6996..83e22fc6a5eb 100644 --- a/core/lib/zksync_core_leftovers/Cargo.toml +++ b/core/lib/zksync_core_leftovers/Cargo.toml @@ -10,106 +10,12 @@ keywords.workspace = true categories.workspace = true [dependencies] -vise.workspace = true -zksync_state.workspace = true -vm_utils.workspace = true -zksync_types.workspace = true zksync_dal.workspace = true -prover_dal.workspace = true -zksync_db_connection.workspace = true zksync_config.workspace = true -zksync_protobuf_config.workspace = true -zksync_utils.workspace = true -zksync_contracts.workspace = true -zksync_system_constants.workspace = true -zksync_eth_client.workspace = true -zksync_eth_signer.workspace = true -zksync_l1_contract_interface.workspace = true -zksync_mempool.workspace = true -zksync_circuit_breaker.workspace = true -zksync_storage.workspace = true -zksync_tee_verifier.workspace = true -zksync_merkle_tree.workspace = true -zksync_mini_merkle_tree.workspace = true -prometheus_exporter.workspace = true -zksync_prover_interface.workspace = true -zksync_queued_job_processor.workspace = true -zksync_web3_decl = { workspace = true, features = ["server"] } -zksync_object_store.workspace = true -zksync_health_check.workspace = true -vlog.workspace = true -zksync_eth_watch.workspace = true -zksync_shared_metrics.workspace = true -zksync_proof_data_handler.workspace = true -zksync_commitment_generator.workspace = true -zksync_house_keeper.workspace = true -zksync_node_genesis.workspace = true -zksync_eth_sender.workspace = true -zksync_node_fee_model.workspace = true -zksync_state_keeper.workspace = true -zksync_metadata_calculator.workspace = true -zksync_node_sync.workspace = true -zksync_node_consensus.workspace = true -zksync_contract_verification_server.workspace = true -zksync_node_api_server.workspace = true -zksync_tee_verifier_input_producer.workspace = true -multivm.workspace = true - -# Consensus dependenices -zksync_concurrency.workspace = true -zksync_consensus_crypto.workspace = true -zksync_consensus_network.workspace = true -zksync_consensus_roles.workspace = true -zksync_consensus_storage.workspace = true -zksync_consensus_executor.workspace = true -zksync_consensus_bft.workspace = true -zksync_consensus_utils.workspace = true zksync_protobuf.workspace = true +zksync_node_genesis.workspace = true -prost.workspace = true -secrecy.workspace = true -serde = { workspace = true, features = ["derive"] } -serde_json.workspace = true +anyhow.workspace = true +tokio = { workspace = true, features = ["time"] } serde_yaml.workspace = true -itertools.workspace = true ctrlc.workspace = true -rand.workspace = true - -tokio = { workspace = true, features = ["time"] } -futures = { workspace = true, features = ["compat"] } -pin-project-lite.workspace = true -chrono = { workspace = true, features = ["serde"] } -anyhow.workspace = true -thiserror.workspace = true -async-trait.workspace = true -thread_local.workspace = true - -reqwest = { workspace = true, features = ["blocking", "json"] } -hex.workspace = true -lru.workspace = true -governor.workspace = true -tower-http = { workspace = true, features = ["full"] } -tower = { workspace = true, features = ["full"] } -axum = { workspace = true, features = [ - "http1", - "json", - "tokio", -] } -once_cell.workspace = true -dashmap.workspace = true - -tracing.workspace = true - -[dev-dependencies] -zksync_test_account.workspace = true -zksync_node_test_utils.workspace = true - -assert_matches.workspace = true -jsonrpsee.workspace = true -tempfile.workspace = true -test-casing.workspace = true -test-log.workspace = true -backon.workspace = true - -[build-dependencies] -zksync_protobuf_build.workspace = true diff --git a/core/lib/zksync_core_leftovers/src/lib.rs b/core/lib/zksync_core_leftovers/src/lib.rs index b4194f99f450..8e85bad9cc33 100644 --- a/core/lib/zksync_core_leftovers/src/lib.rs +++ b/core/lib/zksync_core_leftovers/src/lib.rs @@ -1,80 +1,12 @@ #![allow(clippy::upper_case_acronyms, clippy::derive_partial_eq_without_eq)] -use std::{ - net::Ipv4Addr, - str::FromStr, - sync::Arc, - time::{Duration, Instant}, -}; +use std::str::FromStr; use anyhow::Context as _; -use prometheus_exporter::PrometheusExporterConfig; -use prover_dal::Prover; -use tokio::{ - sync::{oneshot, watch}, - task::JoinHandle, -}; -use zksync_circuit_breaker::{ - l1_txs::FailedL1TransactionChecker, replication_lag::ReplicationLagChecker, - CircuitBreakerChecker, CircuitBreakers, -}; -use zksync_commitment_generator::{ - validation_task::L1BatchCommitmentModeValidationTask, CommitmentGenerator, -}; -use zksync_concurrency::{ctx, scope}; -use zksync_config::{ - configs::{ - api::{MerkleTreeApiConfig, Web3JsonRpcConfig}, - chain::{CircuitBreakerConfig, MempoolConfig, OperationsManagerConfig, StateKeeperConfig}, - consensus::ConsensusConfig, - database::{MerkleTreeConfig, MerkleTreeMode}, - wallets, - wallets::Wallets, - ContractsConfig, DatabaseSecrets, GeneralConfig, Secrets, - }, - ApiConfig, DBConfig, EthWatchConfig, GenesisConfig, -}; -use zksync_contracts::governance_contract; -use zksync_dal::{metrics::PostgresMetrics, ConnectionPool, Core, CoreDal}; -use zksync_db_connection::healthcheck::ConnectionPoolHealthCheck; -use zksync_eth_client::{clients::PKSigningClient, BoundEthInterface}; -use zksync_eth_sender::{Aggregator, EthTxAggregator, EthTxManager}; -use zksync_eth_watch::{EthHttpQueryClient, EthWatch}; -use zksync_health_check::{AppHealthCheck, HealthStatus, ReactiveHealthCheck}; -use zksync_house_keeper::{ - blocks_state_reporter::L1BatchMetricsReporter, - periodic_job::PeriodicJob, - prover::{ - FriGpuProverArchiver, FriProofCompressorJobRetryManager, FriProofCompressorQueueReporter, - FriProverJobRetryManager, FriProverJobsArchiver, FriProverQueueReporter, - FriWitnessGeneratorJobRetryManager, FriWitnessGeneratorQueueReporter, - WaitingToQueuedFriWitnessJobMover, - }, -}; -use zksync_metadata_calculator::{ - api_server::TreeApiHttpClient, MetadataCalculator, MetadataCalculatorConfig, -}; -use zksync_node_api_server::{ - healthcheck::HealthCheckHandle, - tx_sender::{build_tx_sender, TxSenderConfig}, - web3::{self, mempool_cache::MempoolCache, state::InternalApiConfig, Namespace}, -}; -use zksync_node_fee_model::{ - l1_gas_price::GasAdjusterSingleton, BatchFeeModelInputProvider, MainNodeFeeInputProvider, -}; +use tokio::sync::oneshot; +use zksync_config::{configs::DatabaseSecrets, GenesisConfig}; +use zksync_dal::{ConnectionPool, Core, CoreDal as _}; use zksync_node_genesis::{ensure_genesis_state, GenesisParams}; -use zksync_object_store::{ObjectStore, ObjectStoreFactory}; -use zksync_queued_job_processor::JobProcessor; -use zksync_shared_metrics::{InitStage, APP_METRICS}; -use zksync_state::{PostgresStorageCaches, RocksdbStorageOptions}; -use zksync_state_keeper::{ - create_state_keeper, io::seal_logic::l2_block_seal_subtasks::L2BlockSealProcess, - AsyncRocksdbCache, MempoolFetcher, MempoolGuard, OutputHandler, StateKeeperPersistence, - TreeWritesPersistence, -}; -use zksync_tee_verifier_input_producer::TeeVerifierInputProducer; -use zksync_types::{ethabi::Contract, fee_model::FeeModelConfig, Address, L2ChainId}; -use zksync_web3_decl::client::{Client, DynClient, L1}; pub mod temp_config_store; @@ -199,1203 +131,3 @@ impl FromStr for Components { } } } - -pub async fn initialize_components( - configs: &GeneralConfig, - wallets: &Wallets, - genesis_config: &GenesisConfig, - contracts_config: &ContractsConfig, - components: &[Component], - secrets: &Secrets, - consensus_config: Option, -) -> anyhow::Result<( - Vec>>, - watch::Sender, - HealthCheckHandle, -)> { - tracing::info!("Starting the components: {components:?}"); - let l2_chain_id = genesis_config.l2_chain_id; - let db_config = configs.db_config.clone().context("db_config")?; - let postgres_config = configs.postgres_config.clone().context("postgres_config")?; - let database_secrets = secrets.database.clone().context("database_secrets")?; - - if let Some(threshold) = postgres_config.slow_query_threshold() { - ConnectionPool::::global_config().set_slow_query_threshold(threshold)?; - } - if let Some(threshold) = postgres_config.long_connection_threshold() { - ConnectionPool::::global_config().set_long_connection_threshold(threshold)?; - } - - let pool_size = postgres_config.max_connections()?; - let pool_size_master = postgres_config - .max_connections_master() - .unwrap_or(pool_size); - - let connection_pool = - ConnectionPool::::builder(database_secrets.master_url()?, pool_size_master) - .build() - .await - .context("failed to build connection_pool")?; - // We're most interested in setting acquire / statement timeouts for the API server, which puts the most load - // on Postgres. - let replica_connection_pool = - ConnectionPool::::builder(database_secrets.replica_url()?, pool_size) - .set_acquire_timeout(postgres_config.acquire_timeout()) - .set_statement_timeout(postgres_config.statement_timeout()) - .build() - .await - .context("failed to build replica_connection_pool")?; - - let health_check_config = configs - .api_config - .clone() - .context("api_config")? - .healthcheck; - - let app_health = Arc::new(AppHealthCheck::new( - health_check_config.slow_time_limit(), - health_check_config.hard_time_limit(), - )); - - let eth = configs.eth.clone().context("eth")?; - let l1_secrets = secrets.l1.clone().context("l1_secrets")?; - let circuit_breaker_config = configs - .circuit_breaker_config - .clone() - .context("circuit_breaker_config")?; - - let circuit_breaker_checker = CircuitBreakerChecker::new( - Arc::new( - circuit_breakers_for_components(components, &database_secrets, &circuit_breaker_config) - .await - .context("circuit_breakers_for_components")?, - ), - circuit_breaker_config.sync_interval(), - ); - circuit_breaker_checker.check().await.unwrap_or_else(|err| { - panic!("Circuit breaker triggered: {}", err); - }); - - let query_client = Client::http(l1_secrets.l1_rpc_url.clone()) - .context("Ethereum client")? - .for_network(genesis_config.l1_chain_id.into()) - .build(); - let query_client = Box::new(query_client); - let gas_adjuster_config = eth.gas_adjuster.context("gas_adjuster")?; - let sender = eth.sender.as_ref().context("sender")?; - - let mut gas_adjuster = GasAdjusterSingleton::new( - genesis_config.l1_chain_id, - l1_secrets.l1_rpc_url.clone(), - gas_adjuster_config, - sender.pubdata_sending_mode, - genesis_config.l1_batch_commit_data_generator_mode, - ); - - let (stop_sender, stop_receiver) = watch::channel(false); - - // Prometheus exporter and circuit breaker checker should run for every component configuration. - let prom_config = configs - .prometheus_config - .clone() - .context("prometheus_config")?; - let prom_config = PrometheusExporterConfig::pull(prom_config.listener_port); - - let (prometheus_health_check, prometheus_health_updater) = - ReactiveHealthCheck::new("prometheus_exporter"); - app_health.insert_component(prometheus_health_check)?; - let prometheus_task = prom_config.run(stop_receiver.clone()); - let prometheus_task = tokio::spawn(async move { - prometheus_health_updater.update(HealthStatus::Ready.into()); - let res = prometheus_task.await; - drop(prometheus_health_updater); - res - }); - - let mut task_futures: Vec>> = vec![ - prometheus_task, - tokio::spawn(circuit_breaker_checker.run(stop_receiver.clone())), - ]; - - if components.contains(&Component::WsApi) - || components.contains(&Component::HttpApi) - || components.contains(&Component::ContractVerificationApi) - { - let api_config = configs.api_config.clone().context("api_config")?; - let state_keeper_config = configs - .state_keeper_config - .clone() - .context("state_keeper_config")?; - let tx_sender_config = TxSenderConfig::new( - &state_keeper_config, - &api_config.web3_json_rpc, - wallets - .state_keeper - .clone() - .context("Fee account")? - .fee_account - .address(), - l2_chain_id, - ); - let internal_api_config = - InternalApiConfig::new(&api_config.web3_json_rpc, contracts_config, genesis_config); - - // Lazily initialize storage caches only when they are needed (e.g., skip their initialization - // if we only run the explorer APIs). This is required because the cache update task will - // terminate immediately if storage caches are dropped, which will lead to the (unexpected) - // program termination. - let mut storage_caches = None; - - let mempool_cache = MempoolCache::new(api_config.web3_json_rpc.mempool_cache_size()); - let mempool_cache_update_task = mempool_cache.update_task( - connection_pool.clone(), - api_config.web3_json_rpc.mempool_cache_update_interval(), - ); - task_futures.push(tokio::spawn( - mempool_cache_update_task.run(stop_receiver.clone()), - )); - - if components.contains(&Component::HttpApi) { - storage_caches = Some( - build_storage_caches( - &api_config.web3_json_rpc, - &replica_connection_pool, - &mut task_futures, - stop_receiver.clone(), - ) - .context("build_storage_caches()")?, - ); - - let started_at = Instant::now(); - tracing::info!("Initializing HTTP API"); - let bounded_gas_adjuster = gas_adjuster - .get_or_init() - .await - .context("gas_adjuster.get_or_init()")?; - let batch_fee_input_provider = Arc::new(MainNodeFeeInputProvider::new( - bounded_gas_adjuster, - FeeModelConfig::from_state_keeper_config(&state_keeper_config), - )); - run_http_api( - &mut task_futures, - &app_health, - &database_secrets, - &tx_sender_config, - &state_keeper_config, - &internal_api_config, - &api_config, - connection_pool.clone(), - replica_connection_pool.clone(), - stop_receiver.clone(), - batch_fee_input_provider, - state_keeper_config.save_call_traces, - storage_caches.clone().unwrap(), - mempool_cache.clone(), - ) - .await - .context("run_http_api")?; - - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::HttpApi].set(elapsed); - tracing::info!( - "Initialized HTTP API on port {:?} in {elapsed:?}", - api_config.web3_json_rpc.http_port - ); - } - - if components.contains(&Component::WsApi) { - let storage_caches = match storage_caches { - Some(storage_caches) => storage_caches, - None => build_storage_caches( - &configs.api_config.clone().context("api")?.web3_json_rpc, - &replica_connection_pool, - &mut task_futures, - stop_receiver.clone(), - ) - .context("build_storage_caches()")?, - }; - - let started_at = Instant::now(); - tracing::info!("initializing WS API"); - let bounded_gas_adjuster = gas_adjuster - .get_or_init() - .await - .context("gas_adjuster.get_or_init()")?; - let batch_fee_input_provider = Arc::new(MainNodeFeeInputProvider::new( - bounded_gas_adjuster, - FeeModelConfig::from_state_keeper_config(&state_keeper_config), - )); - run_ws_api( - &mut task_futures, - &app_health, - &database_secrets, - &tx_sender_config, - &state_keeper_config, - &internal_api_config, - &api_config, - batch_fee_input_provider, - connection_pool.clone(), - replica_connection_pool.clone(), - stop_receiver.clone(), - storage_caches, - mempool_cache, - ) - .await - .context("run_ws_api")?; - - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::WsApi].set(elapsed); - tracing::info!( - "Initialized WS API on port {} in {elapsed:?}", - api_config.web3_json_rpc.ws_port - ); - } - - if components.contains(&Component::ContractVerificationApi) { - let started_at = Instant::now(); - tracing::info!("initializing contract verification REST API"); - task_futures.push(tokio::spawn( - zksync_contract_verification_server::start_server( - connection_pool.clone(), - replica_connection_pool.clone(), - configs - .contract_verifier - .clone() - .context("Contract verifier")?, - stop_receiver.clone(), - ), - )); - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::ContractVerificationApi].set(elapsed); - tracing::info!("initialized contract verification REST API in {elapsed:?}"); - } - } - - let object_store_config = configs - .core_object_store - .clone() - .context("core_object_store_config")?; - let store_factory = ObjectStoreFactory::new(object_store_config); - - if components.contains(&Component::StateKeeper) { - let started_at = Instant::now(); - tracing::info!("initializing State Keeper"); - let bounded_gas_adjuster = gas_adjuster - .get_or_init() - .await - .context("gas_adjuster.get_or_init()")?; - let state_keeper_config = configs - .state_keeper_config - .clone() - .context("state_keeper_config")?; - let batch_fee_input_provider = Arc::new(MainNodeFeeInputProvider::new( - bounded_gas_adjuster, - FeeModelConfig::from_state_keeper_config(&state_keeper_config), - )); - add_state_keeper_to_task_futures( - &mut task_futures, - &database_secrets, - contracts_config, - state_keeper_config, - wallets - .state_keeper - .clone() - .context("State keeper wallets")?, - l2_chain_id, - &db_config, - &configs.mempool_config.clone().context("mempool_config")?, - batch_fee_input_provider, - stop_receiver.clone(), - ) - .await - .context("add_state_keeper_to_task_futures()")?; - - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::StateKeeper].set(elapsed); - tracing::info!("initialized State Keeper in {elapsed:?}"); - } - - let diamond_proxy_addr = contracts_config.diamond_proxy_addr; - let state_transition_manager_addr = contracts_config - .ecosystem_contracts - .as_ref() - .map(|a| a.state_transition_proxy_addr); - - if components.contains(&Component::Consensus) { - let cfg = consensus_config - .clone() - .context("consensus component's config is missing")?; - let secrets = secrets - .consensus - .clone() - .context("consensus component's secrets are missing")?; - let started_at = Instant::now(); - tracing::info!("initializing Consensus"); - let pool = connection_pool.clone(); - let mut stop_receiver = stop_receiver.clone(); - task_futures.push(tokio::spawn(async move { - // We instantiate the root context here, since the consensus task is the only user of the - // structured concurrency framework. - // Note, however, that awaiting for the `stop_receiver` is related to the root context behavior, - // not the consensus task itself. There may have been any number of tasks running in the root context, - // but we only need to wait for stop signal once, and it will be propagated to all child contexts. - let root_ctx = ctx::root(); - scope::run!(&root_ctx, |ctx, s| async move { - s.spawn_bg(zksync_node_consensus::era::run_main_node( - ctx, cfg, secrets, pool, - )); - let _ = stop_receiver.wait_for(|stop| *stop).await?; - Ok(()) - }) - .await - })); - - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::Consensus].set(elapsed); - tracing::info!("initialized Consensus in {elapsed:?}"); - } - - if components.contains(&Component::EthWatcher) { - let started_at = Instant::now(); - tracing::info!("initializing ETH-Watcher"); - let eth_watch_pool = ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build eth_watch_pool")?; - let governance = (governance_contract(), contracts_config.governance_addr); - let eth_watch_config = configs - .eth - .clone() - .context("eth_config")? - .watcher - .context("watcher")?; - task_futures.push( - start_eth_watch( - eth_watch_config, - eth_watch_pool, - query_client.clone(), - diamond_proxy_addr, - state_transition_manager_addr, - governance, - stop_receiver.clone(), - ) - .await - .context("start_eth_watch()")?, - ); - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::EthWatcher].set(elapsed); - tracing::info!("initialized ETH-Watcher in {elapsed:?}"); - } - - if components.contains(&Component::EthTxAggregator) { - let started_at = Instant::now(); - tracing::info!("initializing ETH-TxAggregator"); - let eth_sender_pool = ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build eth_sender_pool")?; - - let eth_sender_wallets = wallets.eth_sender.clone().context("eth_sender")?; - let operator_private_key = eth_sender_wallets.operator.private_key(); - let diamond_proxy_addr = contracts_config.diamond_proxy_addr; - let default_priority_fee_per_gas = eth - .gas_adjuster - .as_ref() - .context("gas_adjuster")? - .default_priority_fee_per_gas; - let l1_chain_id = genesis_config.l1_chain_id; - - let eth_client = PKSigningClient::new_raw( - operator_private_key.clone(), - diamond_proxy_addr, - default_priority_fee_per_gas, - l1_chain_id, - query_client.clone(), - ); - - let l1_batch_commit_data_generator_mode = - genesis_config.l1_batch_commit_data_generator_mode; - // Run the task synchronously: the main node is expected to have a stable Ethereum client connection, - // and the cost of detecting an incorrect mode with a delay is higher. - L1BatchCommitmentModeValidationTask::new( - contracts_config.diamond_proxy_addr, - l1_batch_commit_data_generator_mode, - query_client.clone(), - ) - .exit_on_success() - .run(stop_receiver.clone()) - .await?; - - let operator_blobs_address = eth_sender_wallets.blob_operator.map(|x| x.address()); - - let sender_config = eth.sender.clone().context("eth_sender")?; - let eth_tx_aggregator_actor = EthTxAggregator::new( - eth_sender_pool, - sender_config.clone(), - Aggregator::new( - sender_config.clone(), - store_factory.create_store().await?, - operator_blobs_address.is_some(), - l1_batch_commit_data_generator_mode, - ), - Box::new(eth_client), - contracts_config.validator_timelock_addr, - contracts_config.l1_multicall3_addr, - diamond_proxy_addr, - l2_chain_id, - operator_blobs_address, - ) - .await; - task_futures.push(tokio::spawn( - eth_tx_aggregator_actor.run(stop_receiver.clone()), - )); - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::EthTxAggregator].set(elapsed); - tracing::info!("initialized ETH-TxAggregator in {elapsed:?}"); - } - - if components.contains(&Component::EthTxManager) { - let started_at = Instant::now(); - tracing::info!("initializing ETH-TxManager"); - let eth_manager_pool = ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build eth_manager_pool")?; - let eth_sender = configs.eth.clone().context("eth_sender_config")?; - let eth_sender_wallets = wallets.eth_sender.clone().context("eth_sender")?; - let operator_private_key = eth_sender_wallets.operator.private_key(); - let diamond_proxy_addr = contracts_config.diamond_proxy_addr; - let default_priority_fee_per_gas = eth - .gas_adjuster - .as_ref() - .context("gas_adjuster")? - .default_priority_fee_per_gas; - let l1_chain_id = genesis_config.l1_chain_id; - - let eth_client = PKSigningClient::new_raw( - operator_private_key.clone(), - diamond_proxy_addr, - default_priority_fee_per_gas, - l1_chain_id, - query_client.clone(), - ); - - let eth_client_blobs = if let Some(blob_operator) = eth_sender_wallets.blob_operator { - let operator_blob_private_key = blob_operator.private_key().clone(); - let client = Box::new(PKSigningClient::new_raw( - operator_blob_private_key, - diamond_proxy_addr, - default_priority_fee_per_gas, - l1_chain_id, - query_client, - )); - Some(client as Box) - } else { - None - }; - - let eth_tx_manager_actor = EthTxManager::new( - eth_manager_pool, - eth_sender.sender.clone().context("eth_sender")?, - gas_adjuster - .get_or_init() - .await - .context("gas_adjuster.get_or_init()")?, - Box::new(eth_client), - eth_client_blobs, - ); - task_futures.extend([tokio::spawn( - eth_tx_manager_actor.run(stop_receiver.clone()), - )]); - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::EthTxManager].set(elapsed); - tracing::info!("initialized ETH-TxManager in {elapsed:?}"); - } - - add_trees_to_task_futures( - configs, - secrets, - &mut task_futures, - &app_health, - components, - &store_factory, - stop_receiver.clone(), - ) - .await - .context("add_trees_to_task_futures()")?; - - if components.contains(&Component::TeeVerifierInputProducer) { - let singleton_connection_pool = - ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build singleton connection_pool")?; - add_tee_verifier_input_producer_to_task_futures( - &mut task_futures, - &singleton_connection_pool, - &store_factory, - l2_chain_id, - stop_receiver.clone(), - ) - .await - .context("add_tee_verifier_input_producer_to_task_futures()")?; - } - - if components.contains(&Component::Housekeeper) { - add_house_keeper_to_task_futures( - configs, - secrets, - &mut task_futures, - stop_receiver.clone(), - ) - .await - .context("add_house_keeper_to_task_futures()")?; - } - - if components.contains(&Component::ProofDataHandler) { - task_futures.push(tokio::spawn(zksync_proof_data_handler::run_server( - configs - .proof_data_handler_config - .clone() - .context("proof_data_handler_config")?, - store_factory.create_store().await?, - connection_pool.clone(), - genesis_config.l1_batch_commit_data_generator_mode, - stop_receiver.clone(), - ))); - } - - if components.contains(&Component::CommitmentGenerator) { - let pool_size = CommitmentGenerator::default_parallelism().get(); - let commitment_generator_pool = - ConnectionPool::::builder(database_secrets.master_url()?, pool_size) - .build() - .await - .context("failed to build commitment_generator_pool")?; - let commitment_generator = CommitmentGenerator::new( - commitment_generator_pool, - genesis_config.l1_batch_commit_data_generator_mode, - ); - app_health.insert_component(commitment_generator.health_check())?; - task_futures.push(tokio::spawn( - commitment_generator.run(stop_receiver.clone()), - )); - } - - // Run healthcheck server for all components. - let db_health_check = ConnectionPoolHealthCheck::new(replica_connection_pool); - app_health.insert_custom_component(Arc::new(db_health_check))?; - let health_check_handle = - HealthCheckHandle::spawn_server(health_check_config.bind_addr(), app_health); - - if let Some(task) = gas_adjuster.run_if_initialized(stop_receiver.clone()) { - task_futures.push(task); - } - - Ok((task_futures, stop_sender, health_check_handle)) -} - -#[allow(clippy::too_many_arguments)] -async fn add_state_keeper_to_task_futures( - task_futures: &mut Vec>>, - database_secrets: &DatabaseSecrets, - contracts_config: &ContractsConfig, - state_keeper_config: StateKeeperConfig, - state_keeper_wallets: wallets::StateKeeper, - l2chain_id: L2ChainId, - db_config: &DBConfig, - mempool_config: &MempoolConfig, - batch_fee_input_provider: Arc, - stop_receiver: watch::Receiver, -) -> anyhow::Result<()> { - let state_keeper_pool = ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build state_keeper_pool")?; - let mempool = { - let mut storage = state_keeper_pool - .connection() - .await - .context("Access storage to build mempool")?; - let mempool = MempoolGuard::from_storage(&mut storage, mempool_config.capacity).await; - mempool.register_metrics(); - mempool - }; - - // L2 Block sealing process is parallelized, so we have to provide enough pooled connections. - let persistence_pool = ConnectionPool::::builder( - database_secrets.master_url()?, - L2BlockSealProcess::subtasks_len(), - ) - .build() - .await - .context("failed to build l2_block_sealer_pool")?; - let (persistence, l2_block_sealer) = StateKeeperPersistence::new( - persistence_pool.clone(), - contracts_config - .l2_shared_bridge_addr - .context("`l2_shared_bridge_addr` config is missing")?, - state_keeper_config.l2_block_seal_queue_capacity, - ); - task_futures.push(tokio::spawn(l2_block_sealer.run())); - - // One (potentially held long-term) connection for `AsyncCatchupTask` and another connection - // to access `AsyncRocksdbCache` as a storage. - let async_cache_pool = ConnectionPool::::builder(database_secrets.master_url()?, 2) - .build() - .await - .context("failed to build async_cache_pool")?; - let cache_options = RocksdbStorageOptions { - block_cache_capacity: db_config - .experimental - .state_keeper_db_block_cache_capacity(), - max_open_files: db_config.experimental.state_keeper_db_max_open_files, - }; - let (async_cache, async_catchup_task) = AsyncRocksdbCache::new( - async_cache_pool, - db_config.state_keeper_db_path.clone(), - cache_options, - ); - - let tree_writes_persistence = TreeWritesPersistence::new(persistence_pool); - let output_handler = - OutputHandler::new(Box::new(persistence)).with_handler(Box::new(tree_writes_persistence)); - let state_keeper = create_state_keeper( - state_keeper_config, - state_keeper_wallets, - async_cache, - l2chain_id, - mempool_config, - state_keeper_pool, - mempool.clone(), - batch_fee_input_provider.clone(), - output_handler, - stop_receiver.clone(), - ) - .await; - - let mut stop_receiver_clone = stop_receiver.clone(); - task_futures.push(tokio::task::spawn(async move { - let result = async_catchup_task.run(stop_receiver_clone.clone()).await; - stop_receiver_clone.changed().await?; - result - })); - task_futures.push(tokio::spawn(state_keeper.run())); - - let mempool_fetcher_pool = ConnectionPool::::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build mempool_fetcher_pool")?; - let mempool_fetcher = MempoolFetcher::new( - mempool, - batch_fee_input_provider, - mempool_config, - mempool_fetcher_pool, - ); - let mempool_fetcher_handle = tokio::spawn(mempool_fetcher.run(stop_receiver)); - task_futures.push(mempool_fetcher_handle); - Ok(()) -} - -pub async fn start_eth_watch( - config: EthWatchConfig, - pool: ConnectionPool, - eth_gateway: Box>, - diamond_proxy_addr: Address, - state_transition_manager_addr: Option
, - governance: (Contract, Address), - stop_receiver: watch::Receiver, -) -> anyhow::Result>> { - let eth_client = EthHttpQueryClient::new( - eth_gateway, - diamond_proxy_addr, - state_transition_manager_addr, - governance.1, - config.confirmations_for_eth_event, - ); - - let eth_watch = EthWatch::new( - diamond_proxy_addr, - &governance.0, - Box::new(eth_client), - pool, - config.poll_interval(), - ) - .await?; - - Ok(tokio::spawn(eth_watch.run(stop_receiver))) -} - -async fn add_trees_to_task_futures( - configs: &GeneralConfig, - secrets: &Secrets, - task_futures: &mut Vec>>, - app_health: &AppHealthCheck, - components: &[Component], - store_factory: &ObjectStoreFactory, - stop_receiver: watch::Receiver, -) -> anyhow::Result<()> { - if !components.contains(&Component::Tree) { - anyhow::ensure!( - !components.contains(&Component::TreeApi), - "Merkle tree API cannot be started without a tree component" - ); - return Ok(()); - } - - let db_config = configs.db_config.clone().context("db_config")?; - let database_secrets = secrets.database.clone().context("database_secrets")?; - let operation_config = configs - .operations_manager_config - .clone() - .context("operations_manager_config")?; - let api_config = configs - .api_config - .clone() - .context("api_config")? - .merkle_tree; - let api_config = components - .contains(&Component::TreeApi) - .then_some(&api_config); - - let object_store = match db_config.merkle_tree.mode { - MerkleTreeMode::Lightweight => None, - MerkleTreeMode::Full => Some(store_factory.create_store().await?), - }; - - run_tree( - task_futures, - app_health, - &database_secrets, - &db_config.merkle_tree, - api_config, - &operation_config, - object_store, - stop_receiver, - ) - .await - .context("run_tree()") -} - -#[allow(clippy::too_many_arguments)] -async fn run_tree( - task_futures: &mut Vec>>, - app_health: &AppHealthCheck, - database_secrets: &DatabaseSecrets, - merkle_tree_config: &MerkleTreeConfig, - api_config: Option<&MerkleTreeApiConfig>, - operation_manager: &OperationsManagerConfig, - object_store: Option>, - stop_receiver: watch::Receiver, -) -> anyhow::Result<()> { - let started_at = Instant::now(); - let mode_str = if matches!(merkle_tree_config.mode, MerkleTreeMode::Full) { - "full" - } else { - "lightweight" - }; - tracing::info!("Initializing Merkle tree in {mode_str} mode"); - - let config = MetadataCalculatorConfig::for_main_node(merkle_tree_config, operation_manager); - let pool = ConnectionPool::singleton(database_secrets.master_url()?) - .build() - .await - .context("failed to build connection pool for Merkle tree")?; - // The number of connections in a recovery pool is based on the mainnet recovery runs. It doesn't need - // to be particularly accurate at this point, since the main node isn't expected to recover from a snapshot. - let recovery_pool = ConnectionPool::builder(database_secrets.replica_url()?, 10) - .build() - .await - .context("failed to build connection pool for Merkle tree recovery")?; - let metadata_calculator = MetadataCalculator::new(config, object_store, pool) - .await - .context("failed initializing metadata_calculator")? - .with_recovery_pool(recovery_pool); - - if let Some(api_config) = api_config { - let address = (Ipv4Addr::UNSPECIFIED, api_config.port).into(); - let tree_reader = metadata_calculator.tree_reader(); - let stop_receiver = stop_receiver.clone(); - task_futures.push(tokio::spawn(async move { - tree_reader - .wait() - .await - .context("Cannot initialize tree reader")? - .run_api_server(address, stop_receiver) - .await - })); - } - - let tree_health_check = metadata_calculator.tree_health_check(); - app_health.insert_custom_component(Arc::new(tree_health_check))?; - let tree_task = tokio::spawn(metadata_calculator.run(stop_receiver)); - task_futures.push(tree_task); - - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::Tree].set(elapsed); - tracing::info!("Initialized {mode_str} tree in {elapsed:?}"); - Ok(()) -} - -async fn add_tee_verifier_input_producer_to_task_futures( - task_futures: &mut Vec>>, - connection_pool: &ConnectionPool, - store_factory: &ObjectStoreFactory, - l2_chain_id: L2ChainId, - stop_receiver: watch::Receiver, -) -> anyhow::Result<()> { - let started_at = Instant::now(); - tracing::info!("initializing TeeVerifierInputProducer"); - let producer = TeeVerifierInputProducer::new( - connection_pool.clone(), - store_factory.create_store().await?, - l2_chain_id, - ) - .await?; - task_futures.push(tokio::spawn(producer.run(stop_receiver, None))); - tracing::info!( - "Initialized TeeVerifierInputProducer in {:?}", - started_at.elapsed() - ); - let elapsed = started_at.elapsed(); - APP_METRICS.init_latency[&InitStage::TeeVerifierInputProducer].set(elapsed); - Ok(()) -} - -async fn add_house_keeper_to_task_futures( - configs: &GeneralConfig, - secrets: &Secrets, - task_futures: &mut Vec>>, - stop_receiver: watch::Receiver, -) -> anyhow::Result<()> { - let house_keeper_config = configs - .house_keeper_config - .clone() - .context("house_keeper_config")?; - let postgres_config = configs.postgres_config.clone().context("postgres_config")?; - let secrets = secrets.database.clone().context("database_secrets")?; - let connection_pool = - ConnectionPool::::builder(secrets.replica_url()?, postgres_config.max_connections()?) - .build() - .await - .context("failed to build a connection pool")?; - - let pool_for_metrics = connection_pool.clone(); - let mut stop_receiver_for_metrics = stop_receiver.clone(); - task_futures.push(tokio::spawn(async move { - tokio::select! { - () = PostgresMetrics::run_scraping(pool_for_metrics, Duration::from_secs(60)) => { - tracing::warn!("Postgres metrics scraping unexpectedly stopped"); - } - _ = stop_receiver_for_metrics.changed() => { - tracing::info!("Stop signal received, Postgres metrics scraping is shutting down"); - } - } - Ok(()) - })); - - let l1_batch_metrics_reporter = L1BatchMetricsReporter::new( - house_keeper_config.l1_batch_metrics_reporting_interval_ms, - connection_pool.clone(), - ); - - let prover_connection_pool = ConnectionPool::::builder( - secrets.prover_url()?, - postgres_config.max_connections()?, - ) - .build() - .await - .context("failed to build a prover_connection_pool")?; - let task = l1_batch_metrics_reporter.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - // All FRI Prover related components are configured below. - let fri_prover_config = configs.prover_config.clone().context("fri_prover_config")?; - let fri_prover_job_retry_manager = FriProverJobRetryManager::new( - fri_prover_config.max_attempts, - fri_prover_config.proof_generation_timeout(), - house_keeper_config.prover_job_retrying_interval_ms, - prover_connection_pool.clone(), - ); - let task = fri_prover_job_retry_manager.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - let fri_witness_gen_config = configs - .witness_generator - .clone() - .context("fri_witness_generator_config")?; - let fri_witness_gen_job_retry_manager = FriWitnessGeneratorJobRetryManager::new( - fri_witness_gen_config.max_attempts, - fri_witness_gen_config.witness_generation_timeouts(), - house_keeper_config.witness_generator_job_retrying_interval_ms, - prover_connection_pool.clone(), - ); - let task = fri_witness_gen_job_retry_manager.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - let waiting_to_queued_fri_witness_job_mover = WaitingToQueuedFriWitnessJobMover::new( - house_keeper_config.witness_job_moving_interval_ms, - prover_connection_pool.clone(), - ); - let task = waiting_to_queued_fri_witness_job_mover.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - let fri_witness_generator_stats_reporter = FriWitnessGeneratorQueueReporter::new( - prover_connection_pool.clone(), - house_keeper_config.witness_generator_stats_reporting_interval_ms, - ); - let task = fri_witness_generator_stats_reporter.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - // TODO(PLA-862): remove after fields become required - if let Some((archiving_interval, archive_after)) = - house_keeper_config.prover_job_archiver_params() - { - let fri_prover_jobs_archiver = FriProverJobsArchiver::new( - prover_connection_pool.clone(), - archiving_interval, - archive_after, - ); - let task = fri_prover_jobs_archiver.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - } - - if let Some((archiving_interval, archive_after)) = - house_keeper_config.fri_gpu_prover_archiver_params() - { - let fri_gpu_prover_jobs_archiver = FriGpuProverArchiver::new( - prover_connection_pool.clone(), - archiving_interval, - archive_after, - ); - let task = fri_gpu_prover_jobs_archiver.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - } - - let fri_prover_group_config = configs - .prover_group_config - .clone() - .context("fri_prover_group_config")?; - let fri_prover_stats_reporter = FriProverQueueReporter::new( - house_keeper_config.prover_stats_reporting_interval_ms, - prover_connection_pool.clone(), - connection_pool.clone(), - fri_prover_group_config, - ); - let task = fri_prover_stats_reporter.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - let proof_compressor_config = configs - .proof_compressor_config - .clone() - .context("fri_proof_compressor_config")?; - let fri_proof_compressor_stats_reporter = FriProofCompressorQueueReporter::new( - house_keeper_config.proof_compressor_stats_reporting_interval_ms, - prover_connection_pool.clone(), - ); - let task = fri_proof_compressor_stats_reporter.run(stop_receiver.clone()); - task_futures.push(tokio::spawn(task)); - - let fri_proof_compressor_retry_manager = FriProofCompressorJobRetryManager::new( - proof_compressor_config.max_attempts, - proof_compressor_config.generation_timeout(), - house_keeper_config.proof_compressor_job_retrying_interval_ms, - prover_connection_pool.clone(), - ); - let task = fri_proof_compressor_retry_manager.run(stop_receiver); - task_futures.push(tokio::spawn(task)); - Ok(()) -} - -fn build_storage_caches( - rpc_config: &Web3JsonRpcConfig, - replica_connection_pool: &ConnectionPool, - task_futures: &mut Vec>>, - stop_receiver: watch::Receiver, -) -> anyhow::Result { - let factory_deps_capacity = rpc_config.factory_deps_cache_size() as u64; - let initial_writes_capacity = rpc_config.initial_writes_cache_size() as u64; - let values_capacity = rpc_config.latest_values_cache_size() as u64; - let mut storage_caches = - PostgresStorageCaches::new(factory_deps_capacity, initial_writes_capacity); - - if values_capacity > 0 { - let values_cache_task = storage_caches - .configure_storage_values_cache(values_capacity, replica_connection_pool.clone()); - task_futures.push(tokio::task::spawn(values_cache_task.run(stop_receiver))); - } - Ok(storage_caches) -} - -#[allow(clippy::too_many_arguments)] -async fn run_http_api( - task_futures: &mut Vec>>, - app_health: &AppHealthCheck, - database_secrets: &DatabaseSecrets, - tx_sender_config: &TxSenderConfig, - state_keeper_config: &StateKeeperConfig, - internal_api: &InternalApiConfig, - api_config: &ApiConfig, - master_connection_pool: ConnectionPool, - replica_connection_pool: ConnectionPool, - stop_receiver: watch::Receiver, - batch_fee_model_input_provider: Arc, - with_debug_namespace: bool, - storage_caches: PostgresStorageCaches, - mempool_cache: MempoolCache, -) -> anyhow::Result<()> { - let (tx_sender, vm_barrier) = build_tx_sender( - tx_sender_config, - &api_config.web3_json_rpc, - state_keeper_config, - replica_connection_pool.clone(), - master_connection_pool, - batch_fee_model_input_provider, - storage_caches, - ) - .await?; - - let mut namespaces = Namespace::DEFAULT.to_vec(); - if with_debug_namespace { - namespaces.push(Namespace::Debug) - } - namespaces.push(Namespace::Snapshots); - - let updaters_pool = ConnectionPool::::builder(database_secrets.replica_url()?, 2) - .build() - .await - .context("failed to build updaters_pool")?; - - let mut api_builder = - web3::ApiBuilder::jsonrpsee_backend(internal_api.clone(), replica_connection_pool) - .http(api_config.web3_json_rpc.http_port) - .with_updaters_pool(updaters_pool) - .with_filter_limit(api_config.web3_json_rpc.filters_limit()) - .with_batch_request_size_limit(api_config.web3_json_rpc.max_batch_request_size()) - .with_response_body_size_limit(api_config.web3_json_rpc.max_response_body_size()) - .with_tx_sender(tx_sender) - .with_vm_barrier(vm_barrier) - .with_mempool_cache(mempool_cache) - .enable_api_namespaces(namespaces); - if let Some(tree_api_url) = api_config.web3_json_rpc.tree_api_url() { - let tree_api = Arc::new(TreeApiHttpClient::new(tree_api_url)); - api_builder = api_builder.with_tree_api(tree_api.clone()); - app_health.insert_custom_component(tree_api)?; - } - - let server_handles = api_builder - .build() - .context("failed to build HTTP API server")? - .run(stop_receiver) - .await?; - task_futures.extend(server_handles.tasks); - app_health.insert_component(server_handles.health_check)?; - Ok(()) -} - -#[allow(clippy::too_many_arguments)] -async fn run_ws_api( - task_futures: &mut Vec>>, - app_health: &AppHealthCheck, - database_secrets: &DatabaseSecrets, - tx_sender_config: &TxSenderConfig, - state_keeper_config: &StateKeeperConfig, - internal_api: &InternalApiConfig, - api_config: &ApiConfig, - batch_fee_model_input_provider: Arc, - master_connection_pool: ConnectionPool, - replica_connection_pool: ConnectionPool, - stop_receiver: watch::Receiver, - storage_caches: PostgresStorageCaches, - mempool_cache: MempoolCache, -) -> anyhow::Result<()> { - let (tx_sender, vm_barrier) = build_tx_sender( - tx_sender_config, - &api_config.web3_json_rpc, - state_keeper_config, - replica_connection_pool.clone(), - master_connection_pool, - batch_fee_model_input_provider, - storage_caches, - ) - .await?; - let updaters_pool = ConnectionPool::::singleton(database_secrets.replica_url()?) - .build() - .await - .context("failed to build updaters_pool")?; - - let mut namespaces = Namespace::DEFAULT.to_vec(); - namespaces.push(Namespace::Snapshots); - - let mut api_builder = - web3::ApiBuilder::jsonrpsee_backend(internal_api.clone(), replica_connection_pool) - .ws(api_config.web3_json_rpc.ws_port) - .with_updaters_pool(updaters_pool) - .with_filter_limit(api_config.web3_json_rpc.filters_limit()) - .with_subscriptions_limit(api_config.web3_json_rpc.subscriptions_limit()) - .with_batch_request_size_limit(api_config.web3_json_rpc.max_batch_request_size()) - .with_response_body_size_limit(api_config.web3_json_rpc.max_response_body_size()) - .with_websocket_requests_per_minute_limit( - api_config - .web3_json_rpc - .websocket_requests_per_minute_limit(), - ) - .with_polling_interval(api_config.web3_json_rpc.pubsub_interval()) - .with_tx_sender(tx_sender) - .with_vm_barrier(vm_barrier) - .with_mempool_cache(mempool_cache) - .enable_api_namespaces(namespaces); - if let Some(tree_api_url) = api_config.web3_json_rpc.tree_api_url() { - let tree_api = Arc::new(TreeApiHttpClient::new(tree_api_url)); - api_builder = api_builder.with_tree_api(tree_api.clone()); - app_health.insert_custom_component(tree_api)?; - } - - let server_handles = api_builder - .build() - .context("failed to build WS API server")? - .run(stop_receiver) - .await?; - task_futures.extend(server_handles.tasks); - app_health.insert_component(server_handles.health_check)?; - Ok(()) -} - -async fn circuit_breakers_for_components( - components: &[Component], - database_secrets: &DatabaseSecrets, - circuit_breaker_config: &CircuitBreakerConfig, -) -> anyhow::Result { - let circuit_breakers = CircuitBreakers::default(); - - if components - .iter() - .any(|c| matches!(c, Component::EthTxAggregator | Component::EthTxManager)) - { - let pool = ConnectionPool::::singleton(database_secrets.replica_url()?) - .build() - .await - .context("failed to build a connection pool")?; - circuit_breakers - .insert(Box::new(FailedL1TransactionChecker { pool })) - .await; - } - - if components.iter().any(|c| { - matches!( - c, - Component::HttpApi | Component::WsApi | Component::ContractVerificationApi - ) - }) { - let pool = ConnectionPool::::singleton(database_secrets.replica_url()?) - .build() - .await?; - circuit_breakers - .insert(Box::new(ReplicationLagChecker { - pool, - replication_lag_limit: circuit_breaker_config.replication_lag_limit(), - })) - .await; - } - Ok(circuit_breakers) -} diff --git a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs index 1f4c410ed9c1..cb3e0d08794d 100644 --- a/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs +++ b/core/lib/zksync_core_leftovers/src/temp_config_store/mod.rs @@ -15,20 +15,14 @@ use zksync_config::{ ApiConfig, ContractVerifierConfig, DBConfig, EthConfig, EthWatchConfig, GasAdjusterConfig, ObjectStoreConfig, PostgresConfig, SnapshotsCreatorConfig, }; -use zksync_protobuf::{repr::ProtoRepr, ProtoFmt}; - -pub fn decode_yaml(yaml: &str) -> anyhow::Result { - let d = serde_yaml::Deserializer::from_str(yaml); - let this: T = zksync_protobuf::serde::deserialize(d)?; - Ok(this) -} +use zksync_protobuf::repr::ProtoRepr; pub fn decode_yaml_repr(yaml: &str) -> anyhow::Result { let d = serde_yaml::Deserializer::from_str(yaml); let this: T = zksync_protobuf::serde::deserialize_proto_with_options(d, false)?; this.read() } -// + // TODO (QIT-22): This structure is going to be removed when components will be responsible for their own configs. /// A temporary config store allowing to pass deserialized configs from `zksync_server` to `zksync_core`. /// All the configs are optional, since for some component combination it is not needed to pass all the configs. diff --git a/infrastructure/zk/src/server.ts b/infrastructure/zk/src/server.ts index 872aff2eb5c3..2ed74deca98e 100644 --- a/infrastructure/zk/src/server.ts +++ b/infrastructure/zk/src/server.ts @@ -21,9 +21,6 @@ export async function server(rebuildTree: boolean, uring: boolean, components?: if (components) { options += ` --components=${components}`; } - if (useNodeFramework) { - options += ' --use-node-framework'; - } await utils.spawn(`cargo run --bin zksync_server --release ${options}`); } @@ -82,7 +79,6 @@ export const serverCommand = new Command('server') .option('--uring', 'enables uring support for RocksDB') .option('--components ', 'comma-separated list of components to run') .option('--chain-name ', 'environment name') - .option('--use-node-framework', 'use node framework for server') .action(async (cmd: Command) => { cmd.chainName ? env.reload(cmd.chainName) : env.load(); if (cmd.genesis) { diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 6d6b967fbbe7..c0e965605fd4 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -28,41 +28,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" -[[package]] -name = "aead" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" -dependencies = [ - "crypto-common", - "generic-array", -] - -[[package]] -name = "aes" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" -dependencies = [ - "cfg-if 1.0.0", - "cipher", - "cpufeatures", -] - -[[package]] -name = "aes-gcm" -version = "0.10.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" -dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", - "subtle", -] - [[package]] name = "ahash" version = "0.7.8" @@ -96,21 +61,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "alloc-no-stdlib" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" - -[[package]] -name = "alloc-stdlib" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" -dependencies = [ - "alloc-no-stdlib", -] - [[package]] name = "allocator-api2" version = "0.2.18" @@ -244,22 +194,6 @@ version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" -[[package]] -name = "async-compression" -version = "0.4.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" -dependencies = [ - "brotli", - "flate2", - "futures-core", - "memchr", - "pin-project-lite", - "tokio", - "zstd", - "zstd-safe", -] - [[package]] name = "async-lock" version = "3.4.0" @@ -352,11 +286,7 @@ dependencies = [ "pin-project-lite", "rustversion", "serde", - "serde_json", - "serde_path_to_error", - "serde_urlencoded", "sync_wrapper", - "tokio", "tower", "tower-layer", "tower-service", @@ -605,15 +535,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bitmaps" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "031043d04099746d8db04daf1fa424b2bc8bd69d92b25962dcde24da39ab64a2" -dependencies = [ - "typenum", -] - [[package]] name = "bitvec" version = "1.0.1" @@ -801,27 +722,6 @@ dependencies = [ "syn_derive", ] -[[package]] -name = "brotli" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", - "brotli-decompressor", -] - -[[package]] -name = "brotli-decompressor" -version = "4.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" -dependencies = [ - "alloc-no-stdlib", - "alloc-stdlib", -] - [[package]] name = "bumpalo" version = "3.16.0" @@ -954,30 +854,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" -[[package]] -name = "chacha20" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" -dependencies = [ - "cfg-if 1.0.0", - "cipher", - "cpufeatures", -] - -[[package]] -name = "chacha20poly1305" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" -dependencies = [ - "aead", - "chacha20", - "cipher", - "poly1305", - "zeroize", -] - [[package]] name = "chrono" version = "0.4.38" @@ -993,17 +869,6 @@ dependencies = [ "windows-targets 0.52.5", ] -[[package]] -name = "cipher" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" -dependencies = [ - "crypto-common", - "inout", - "zeroize", -] - [[package]] name = "circuit_definitions" version = "0.1.0" @@ -1221,22 +1086,6 @@ dependencies = [ "cc", ] -[[package]] -name = "codegen" -version = "0.1.0" -source = "git+https://github.com/matter-labs/solidity_plonk_verifier.git?branch=dev#82f96b7156551087f1c9bfe4f0ea68845b6debfc" -dependencies = [ - "ethereum-types", - "franklin-crypto 0.0.5 (git+https://github.com/matter-labs/franklin-crypto?branch=dev)", - "handlebars", - "hex", - "paste", - "rescue_poseidon 0.4.1 (git+https://github.com/matter-labs/rescue-poseidon.git)", - "serde", - "serde_derive", - "serde_json", -] - [[package]] name = "codegen" version = "0.2.0" @@ -1561,7 +1410,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ "generic-array", - "rand_core 0.6.4", "typenum", ] @@ -1598,15 +1446,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "ctr" -version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835" -dependencies = [ - "cipher", -] - [[package]] name = "ctrlc" version = "3.4.4" @@ -2506,20 +2345,10 @@ dependencies = [ "cfg-if 1.0.0", "js-sys", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] -[[package]] -name = "ghash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" -dependencies = [ - "opaque-debug", - "polyval", -] - [[package]] name = "gimli" version = "0.29.0" @@ -2651,23 +2480,6 @@ dependencies = [ "async-trait", ] -[[package]] -name = "governor" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19775995ee20209163239355bc3ad2f33f83da35d9ef72dea26e5af753552c87" -dependencies = [ - "dashmap", - "futures 0.3.30", - "futures-timer", - "no-std-compat", - "nonzero_ext", - "parking_lot", - "quanta 0.9.3", - "rand 0.8.5", - "smallvec", -] - [[package]] name = "gpu-ffi" version = "0.1.0" @@ -2738,20 +2550,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "handlebars" -version = "5.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08485b96a0e6393e9e4d1b8d48cf74ad6c063cd905eb33f42c1ce3f0377539b" -dependencies = [ - "log", - "pest", - "pest_derive", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "hashbrown" version = "0.12.3" @@ -2789,16 +2587,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "hdrhistogram" -version = "7.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" -dependencies = [ - "byteorder", - "num-traits", -] - [[package]] name = "heck" version = "0.3.3" @@ -2904,12 +2692,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "http-range-header" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "add0ab9360ddbd88cfeb3bd9574a1d85cfdfa14db10b3e21d3700dbc4328758f" - [[package]] name = "httparse" version = "1.8.0" @@ -3032,20 +2814,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "im" -version = "15.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0acd33ff0285af998aaf9b57342af478078f53492322fafc47450e09397e0e9" -dependencies = [ - "bitmaps", - "rand_core 0.6.4", - "rand_xoshiro", - "sized-chunks", - "typenum", - "version_check", -] - [[package]] name = "impl-codec" version = "0.6.0" @@ -3116,15 +2884,6 @@ dependencies = [ "regex", ] -[[package]] -name = "inout" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" -dependencies = [ - "generic-array", -] - [[package]] name = "ipnet" version = "2.9.0" @@ -3140,16 +2899,6 @@ dependencies = [ "serde", ] -[[package]] -name = "iri-string" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f5f6c2df22c009ac44f6f1499308e7a3ac7ba42cd2378475cc691510e1eef1b" -dependencies = [ - "memchr", - "serde", -] - [[package]] name = "is_terminal_polyfill" version = "1.70.0" @@ -3237,11 +2986,9 @@ dependencies = [ "jsonrpsee-core", "jsonrpsee-http-client", "jsonrpsee-proc-macros", - "jsonrpsee-server", "jsonrpsee-types", "jsonrpsee-wasm-client", "jsonrpsee-ws-client", - "tokio", "tracing", ] @@ -3283,9 +3030,7 @@ dependencies = [ "futures-util", "hyper", "jsonrpsee-types", - "parking_lot", "pin-project", - "rand 0.8.5", "rustc-hash", "serde", "serde_json", @@ -3329,30 +3074,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "jsonrpsee-server" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cc7c6d1a2c58f6135810284a390d9f823d0f508db74cd914d8237802de80f98" -dependencies = [ - "futures-util", - "http", - "hyper", - "jsonrpsee-core", - "jsonrpsee-types", - "pin-project", - "route-recognizer", - "serde", - "serde_json", - "soketto", - "thiserror", - "tokio", - "tokio-stream", - "tokio-util", - "tower", - "tracing", -] - [[package]] name = "jsonrpsee-types" version = "0.21.0" @@ -3624,12 +3345,6 @@ dependencies = [ "logos-codegen", ] -[[package]] -name = "lru" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc" - [[package]] name = "lz4-sys" version = "1.9.4" @@ -3640,15 +3355,6 @@ dependencies = [ "libc", ] -[[package]] -name = "mach" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b823e83b2affd8f40a9ee8c29dbc56404c1e34cd2710921f2801e2cf29527afa" -dependencies = [ - "libc", -] - [[package]] name = "mach2" version = "0.4.2" @@ -3739,7 +3445,7 @@ dependencies = [ "ipnet", "metrics", "metrics-util", - "quanta 0.11.1", + "quanta", "thiserror", "tokio", "tracing", @@ -3767,7 +3473,7 @@ dependencies = [ "hashbrown 0.13.1", "metrics", "num_cpus", - "quanta 0.11.1", + "quanta", "sketches-ddsketch", ] @@ -3847,7 +3553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.48.0", ] @@ -3940,12 +3646,6 @@ dependencies = [ "libc", ] -[[package]] -name = "no-std-compat" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" - [[package]] name = "nodrop" version = "0.1.14" @@ -3962,12 +3662,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nonzero_ext" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" - [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -4524,51 +4218,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" -[[package]] -name = "pest" -version = "2.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "560131c633294438da9f7c4b08189194b20946c8274c6b9e38881a7874dc8ee8" -dependencies = [ - "memchr", - "thiserror", - "ucd-trie", -] - -[[package]] -name = "pest_derive" -version = "2.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26293c9193fbca7b1a3bf9b79dc1e388e927e6cacaa78b4a3ab705a1d3d41459" -dependencies = [ - "pest", - "pest_generator", -] - -[[package]] -name = "pest_generator" -version = "2.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ec22af7d3fb470a85dd2ca96b7c577a1eb4ef6f1683a9fe9a8c16e136c04687" -dependencies = [ - "pest", - "pest_meta", - "proc-macro2 1.0.85", - "quote 1.0.36", - "syn 2.0.66", -] - -[[package]] -name = "pest_meta" -version = "2.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7a240022f37c361ec1878d646fc5b7d7c4d28d5946e1a80ad5a7a4f4ca0bdcd" -dependencies = [ - "once_cell", - "pest", - "sha2 0.10.8", -] - [[package]] name = "petgraph" version = "0.6.5" @@ -4648,29 +4297,6 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" -[[package]] -name = "poly1305" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8159bd90725d2df49889a078b54f4f79e87f1f8a8444194cdca81d38f5393abf" -dependencies = [ - "cpufeatures", - "opaque-debug", - "universal-hash", -] - -[[package]] -name = "polyval" -version = "0.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" -dependencies = [ - "cfg-if 1.0.0", - "cpufeatures", - "opaque-debug", - "universal-hash", -] - [[package]] name = "portable-atomic" version = "1.6.0" @@ -5043,22 +4669,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "quanta" -version = "0.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20afe714292d5e879d8b12740aa223c6a88f118af41870e8b6196e39a02238a8" -dependencies = [ - "crossbeam-utils 0.8.20", - "libc", - "mach", - "once_cell", - "raw-cpuid", - "wasi 0.10.2+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - [[package]] name = "quanta" version = "0.11.1" @@ -5070,7 +4680,7 @@ dependencies = [ "mach2", "once_cell", "raw-cpuid", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -5187,15 +4797,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "rand_xoshiro" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "raw-cpuid" version = "10.7.0" @@ -5477,12 +5078,6 @@ dependencies = [ "librocksdb-sys", ] -[[package]] -name = "route-recognizer" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" - [[package]] name = "rsa" version = "0.9.6" @@ -5968,16 +5563,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_path_to_error" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" -dependencies = [ - "itoa", - "serde", -] - [[package]] name = "serde_urlencoded" version = "0.7.1" @@ -6204,16 +5789,6 @@ dependencies = [ "time", ] -[[package]] -name = "sized-chunks" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16d69225bde7a69b235da73377861095455d298f2b970996eec25ddbb42b3d1e" -dependencies = [ - "bitmaps", - "typenum", -] - [[package]] name = "skeptic" version = "0.13.7" @@ -6263,22 +5838,6 @@ dependencies = [ "rescue_poseidon 0.4.1 (git+https://github.com/matter-labs/rescue-poseidon.git?branch=poseidon2)", ] -[[package]] -name = "snow" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "850948bee068e713b8ab860fe1adc4d109676ab4c3b621fd8147f06b261f2f85" -dependencies = [ - "aes-gcm", - "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", - "chacha20poly1305", - "curve25519-dalek", - "rand_core 0.6.4", - "rustc_version", - "sha2 0.10.8", - "subtle", -] - [[package]] name = "socket2" version = "0.5.7" @@ -6298,7 +5857,6 @@ dependencies = [ "base64 0.13.1", "bytes", "futures 0.3.30", - "http", "httparse", "log", "rand 0.8.5", @@ -7091,7 +6649,6 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "hdrhistogram", "indexmap 1.9.3", "pin-project", "pin-project-lite", @@ -7104,36 +6661,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "tower-http" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c5bb1d698276a2443e5ecfabc1008bf15a36c12e6a7176e7bf089ea9131140" -dependencies = [ - "async-compression", - "base64 0.21.7", - "bitflags 2.5.0", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-range-header", - "httpdate", - "iri-string", - "mime", - "mime_guess", - "percent-encoding", - "pin-project-lite", - "tokio", - "tokio-util", - "tower", - "tower-layer", - "tower-service", - "tracing", - "uuid", -] - [[package]] name = "tower-layer" version = "0.3.2" @@ -7273,12 +6800,6 @@ version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" -[[package]] -name = "ucd-trie" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" - [[package]] name = "uint" version = "0.9.5" @@ -7372,16 +6893,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "universal-hash" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" -dependencies = [ - "crypto-common", - "subtle", -] - [[package]] name = "unroll" version = "0.1.5" @@ -7447,7 +6958,6 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" dependencies = [ - "getrandom", "serde", ] @@ -7559,21 +7069,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "vm_utils" -version = "0.1.0" -dependencies = [ - "anyhow", - "multivm", - "tokio", - "tracing", - "zksync_contracts", - "zksync_dal", - "zksync_state", - "zksync_types", - "zksync_utils", -] - [[package]] name = "wait-timeout" version = "0.2.0" @@ -7602,12 +7097,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -8307,7 +7796,7 @@ dependencies = [ "bincode", "circuit_sequencer_api 0.1.0", "circuit_testing", - "codegen 0.2.0", + "codegen", "crossbeam 0.8.4", "derivative", "env_logger 0.11.3", @@ -8334,7 +7823,7 @@ source = "git+https://github.com/matter-labs/era-zkevm_test_harness?branch=gpu-w dependencies = [ "bincode", "circuit_definitions 0.1.0", - "codegen 0.2.0", + "codegen", "crossbeam 0.8.4", "derivative", "env_logger 0.11.3", @@ -8358,7 +7847,7 @@ dependencies = [ "bincode", "circuit_definitions 1.5.0", "circuit_sequencer_api 0.1.50", - "codegen 0.2.0", + "codegen", "crossbeam 0.8.4", "curl", "derivative", @@ -8398,48 +7887,6 @@ dependencies = [ "url", ] -[[package]] -name = "zksync_circuit_breaker" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "thiserror", - "tokio", - "tracing", - "vise", - "zksync_config", - "zksync_dal", -] - -[[package]] -name = "zksync_commitment_generator" -version = "0.1.0" -dependencies = [ - "anyhow", - "circuit_sequencer_api 0.1.40", - "circuit_sequencer_api 0.1.41", - "circuit_sequencer_api 0.1.50", - "futures 0.3.30", - "itertools 0.10.5", - "multivm", - "num_cpus", - "serde_json", - "tokio", - "tracing", - "vise", - "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.3-rc2)", - "zk_evm 1.4.1", - "zk_evm 1.5.0", - "zksync_contracts", - "zksync_dal", - "zksync_eth_client", - "zksync_health_check", - "zksync_l1_contract_interface", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_concurrency" version = "0.1.0" @@ -8471,27 +7918,6 @@ dependencies = [ "zksync_crypto_primitives", ] -[[package]] -name = "zksync_consensus_bft" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3e6f101ee4124308c4c974caaa259d524549b0c6#3e6f101ee4124308c4c974caaa259d524549b0c6" -dependencies = [ - "anyhow", - "async-trait", - "once_cell", - "rand 0.8.5", - "thiserror", - "tracing", - "vise", - "zksync_concurrency", - "zksync_consensus_crypto", - "zksync_consensus_network", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_protobuf", -] - [[package]] name = "zksync_consensus_crypto" version = "0.1.0" @@ -8513,50 +7939,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "zksync_consensus_executor" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3e6f101ee4124308c4c974caaa259d524549b0c6#3e6f101ee4124308c4c974caaa259d524549b0c6" -dependencies = [ - "anyhow", - "rand 0.8.5", - "tracing", - "vise", - "zksync_concurrency", - "zksync_consensus_bft", - "zksync_consensus_crypto", - "zksync_consensus_network", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_protobuf", -] - -[[package]] -name = "zksync_consensus_network" -version = "0.1.0" -source = "git+https://github.com/matter-labs/era-consensus.git?rev=3e6f101ee4124308c4c974caaa259d524549b0c6#3e6f101ee4124308c4c974caaa259d524549b0c6" -dependencies = [ - "anyhow", - "async-trait", - "im", - "once_cell", - "pin-project", - "prost 0.12.6", - "rand 0.8.5", - "snow", - "thiserror", - "tracing", - "vise", - "zksync_concurrency", - "zksync_consensus_crypto", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_protobuf", - "zksync_protobuf_build", -] - [[package]] name = "zksync_consensus_roles" version = "0.1.0" @@ -8607,142 +7989,53 @@ dependencies = [ ] [[package]] -name = "zksync_contract_verification_server" +name = "zksync_contracts" version = "0.1.0" dependencies = [ - "anyhow", - "axum", + "envy", + "ethabi", + "hex", + "once_cell", "serde", "serde_json", + "zksync_utils", +] + +[[package]] +name = "zksync_core_leftovers" +version = "0.1.0" +dependencies = [ + "anyhow", + "ctrlc", + "serde_yaml", "tokio", - "tower-http", - "tracing", - "vise", "zksync_config", "zksync_dal", - "zksync_types", + "zksync_node_genesis", + "zksync_protobuf", ] [[package]] -name = "zksync_contracts" +name = "zksync_crypto" version = "0.1.0" dependencies = [ - "envy", - "ethabi", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "once_cell", "serde", - "serde_json", - "zksync_utils", + "sha2 0.10.8", + "thiserror", + "zksync_basic_types", ] [[package]] -name = "zksync_core_leftovers" +name = "zksync_crypto_primitives" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", - "axum", - "chrono", - "ctrlc", - "dashmap", - "futures 0.3.30", - "governor", "hex", - "itertools 0.10.5", - "lru", - "multivm", - "once_cell", - "pin-project-lite", - "prometheus_exporter", - "prost 0.12.6", - "prover_dal", "rand 0.8.5", - "reqwest", - "secrecy", - "serde", - "serde_json", - "serde_yaml", - "thiserror", - "thread_local", - "tokio", - "tower", - "tower-http", - "tracing", - "vise", - "vlog", - "vm_utils", - "zksync_circuit_breaker", - "zksync_commitment_generator", - "zksync_concurrency", - "zksync_config", - "zksync_consensus_bft", - "zksync_consensus_crypto", - "zksync_consensus_executor", - "zksync_consensus_network", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_contract_verification_server", - "zksync_contracts", - "zksync_dal", - "zksync_db_connection", - "zksync_eth_client", - "zksync_eth_sender", - "zksync_eth_signer", - "zksync_eth_watch", - "zksync_health_check", - "zksync_house_keeper", - "zksync_l1_contract_interface", - "zksync_mempool", - "zksync_merkle_tree", - "zksync_metadata_calculator", - "zksync_mini_merkle_tree", - "zksync_node_api_server", - "zksync_node_consensus", - "zksync_node_fee_model", - "zksync_node_genesis", - "zksync_node_sync", - "zksync_object_store", - "zksync_proof_data_handler", - "zksync_protobuf", - "zksync_protobuf_build", - "zksync_protobuf_config", - "zksync_prover_interface", - "zksync_queued_job_processor", - "zksync_shared_metrics", - "zksync_state", - "zksync_state_keeper", - "zksync_storage", - "zksync_system_constants", - "zksync_tee_verifier", - "zksync_tee_verifier_input_producer", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", -] - -[[package]] -name = "zksync_crypto" -version = "0.1.0" -dependencies = [ - "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", - "hex", - "once_cell", - "serde", - "sha2 0.10.8", - "thiserror", - "zksync_basic_types", -] - -[[package]] -name = "zksync_crypto_primitives" -version = "0.1.0" -dependencies = [ - "anyhow", - "hex", - "rand 0.8.5", - "secp256k1", + "secp256k1", "serde", "serde_json", "thiserror", @@ -8826,30 +8119,6 @@ dependencies = [ "zksync_web3_decl", ] -[[package]] -name = "zksync_eth_sender" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "thiserror", - "tokio", - "tracing", - "vise", - "zksync_config", - "zksync_contracts", - "zksync_dal", - "zksync_eth_client", - "zksync_l1_contract_interface", - "zksync_node_fee_model", - "zksync_object_store", - "zksync_prover_interface", - "zksync_shared_metrics", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_eth_signer" version = "0.1.0" @@ -8860,24 +8129,6 @@ dependencies = [ "zksync_types", ] -[[package]] -name = "zksync_eth_watch" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "thiserror", - "tokio", - "tracing", - "vise", - "zksync_contracts", - "zksync_dal", - "zksync_eth_client", - "zksync_shared_metrics", - "zksync_system_constants", - "zksync_types", -] - [[package]] name = "zksync_health_check" version = "0.1.0" @@ -8892,44 +8143,6 @@ dependencies = [ "vise", ] -[[package]] -name = "zksync_house_keeper" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "prover_dal", - "tokio", - "tracing", - "vise", - "zksync_config", - "zksync_dal", - "zksync_shared_metrics", - "zksync_types", -] - -[[package]] -name = "zksync_l1_contract_interface" -version = "0.1.0" -dependencies = [ - "codegen 0.1.0", - "hex", - "kzg", - "once_cell", - "sha2 0.10.8", - "sha3 0.10.8", - "zksync_prover_interface", - "zksync_types", -] - -[[package]] -name = "zksync_mempool" -version = "0.1.0" -dependencies = [ - "tracing", - "zksync_types", -] - [[package]] name = "zksync_merkle_tree" version = "0.1.0" @@ -8949,35 +8162,6 @@ dependencies = [ "zksync_utils", ] -[[package]] -name = "zksync_metadata_calculator" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "futures 0.3.30", - "itertools 0.10.5", - "once_cell", - "reqwest", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "vise", - "zksync_config", - "zksync_crypto", - "zksync_dal", - "zksync_health_check", - "zksync_merkle_tree", - "zksync_object_store", - "zksync_shared_metrics", - "zksync_storage", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_mini_merkle_tree" version = "0.1.0" @@ -8987,100 +8171,6 @@ dependencies = [ "zksync_crypto", ] -[[package]] -name = "zksync_node_api_server" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "axum", - "chrono", - "futures 0.3.30", - "governor", - "hex", - "http", - "itertools 0.10.5", - "lru", - "multivm", - "once_cell", - "pin-project-lite", - "rand 0.8.5", - "serde", - "serde_json", - "thiserror", - "thread_local", - "tokio", - "tower", - "tower-http", - "tracing", - "vise", - "zksync_config", - "zksync_contracts", - "zksync_dal", - "zksync_health_check", - "zksync_metadata_calculator", - "zksync_mini_merkle_tree", - "zksync_node_fee_model", - "zksync_node_sync", - "zksync_protobuf", - "zksync_shared_metrics", - "zksync_state", - "zksync_state_keeper", - "zksync_system_constants", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", -] - -[[package]] -name = "zksync_node_consensus" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "secrecy", - "tempfile", - "tracing", - "zksync_concurrency", - "zksync_config", - "zksync_consensus_bft", - "zksync_consensus_crypto", - "zksync_consensus_executor", - "zksync_consensus_network", - "zksync_consensus_roles", - "zksync_consensus_storage", - "zksync_consensus_utils", - "zksync_dal", - "zksync_l1_contract_interface", - "zksync_merkle_tree", - "zksync_metadata_calculator", - "zksync_node_sync", - "zksync_protobuf", - "zksync_state", - "zksync_state_keeper", - "zksync_system_constants", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", -] - -[[package]] -name = "zksync_node_fee_model" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "tokio", - "tracing", - "vise", - "zksync_config", - "zksync_dal", - "zksync_eth_client", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", -] - [[package]] name = "zksync_node_genesis" version = "0.1.0" @@ -9102,50 +8192,6 @@ dependencies = [ "zksync_utils", ] -[[package]] -name = "zksync_node_sync" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "chrono", - "futures 0.3.30", - "serde", - "serde_json", - "thiserror", - "tokio", - "tracing", - "vise", - "vm_utils", - "zksync_concurrency", - "zksync_config", - "zksync_contracts", - "zksync_dal", - "zksync_eth_client", - "zksync_health_check", - "zksync_node_genesis", - "zksync_shared_metrics", - "zksync_state_keeper", - "zksync_system_constants", - "zksync_types", - "zksync_utils", - "zksync_web3_decl", -] - -[[package]] -name = "zksync_node_test_utils" -version = "0.1.0" -dependencies = [ - "multivm", - "zksync_contracts", - "zksync_dal", - "zksync_merkle_tree", - "zksync_node_genesis", - "zksync_system_constants", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_object_store" version = "0.1.0" @@ -9169,22 +8215,6 @@ dependencies = [ "zksync_types", ] -[[package]] -name = "zksync_proof_data_handler" -version = "0.1.0" -dependencies = [ - "anyhow", - "axum", - "tokio", - "tracing", - "zksync_config", - "zksync_dal", - "zksync_object_store", - "zksync_prover_interface", - "zksync_tee_verifier", - "zksync_types", -] - [[package]] name = "zksync_proof_fri_compressor" version = "0.1.0" @@ -9429,38 +8459,6 @@ dependencies = [ "zksync_utils", ] -[[package]] -name = "zksync_state_keeper" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "futures 0.3.30", - "hex", - "itertools 0.10.5", - "multivm", - "once_cell", - "thiserror", - "tokio", - "tracing", - "vise", - "vm_utils", - "zksync_config", - "zksync_contracts", - "zksync_dal", - "zksync_mempool", - "zksync_node_fee_model", - "zksync_node_genesis", - "zksync_node_test_utils", - "zksync_protobuf", - "zksync_shared_metrics", - "zksync_state", - "zksync_storage", - "zksync_test_account", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_storage" version = "0.1.0" @@ -9482,61 +8480,6 @@ dependencies = [ "zksync_utils", ] -[[package]] -name = "zksync_tee_verifier" -version = "0.1.0" -dependencies = [ - "anyhow", - "multivm", - "serde", - "tracing", - "vm_utils", - "zksync_config", - "zksync_crypto", - "zksync_dal", - "zksync_db_connection", - "zksync_merkle_tree", - "zksync_object_store", - "zksync_prover_interface", - "zksync_queued_job_processor", - "zksync_state", - "zksync_types", - "zksync_utils", -] - -[[package]] -name = "zksync_tee_verifier_input_producer" -version = "0.1.0" -dependencies = [ - "anyhow", - "async-trait", - "tokio", - "tracing", - "vise", - "vm_utils", - "zksync_dal", - "zksync_object_store", - "zksync_prover_interface", - "zksync_queued_job_processor", - "zksync_tee_verifier", - "zksync_types", - "zksync_utils", -] - -[[package]] -name = "zksync_test_account" -version = "0.1.0" -dependencies = [ - "ethabi", - "hex", - "rand 0.8.5", - "zksync_contracts", - "zksync_eth_signer", - "zksync_system_constants", - "zksync_types", - "zksync_utils", -] - [[package]] name = "zksync_types" version = "0.1.0" @@ -9683,24 +8626,6 @@ dependencies = [ "zksync_utils", ] -[[package]] -name = "zstd" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" -dependencies = [ - "zstd-safe", -] - -[[package]] -name = "zstd-safe" -version = "7.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" -dependencies = [ - "zstd-sys", -] - [[package]] name = "zstd-sys" version = "2.0.10+zstd.1.5.6" From d5e8e9bc66ff38b828730b62d8a7b8794cb1758a Mon Sep 17 00:00:00 2001 From: AnastasiiaVashchuk <72273339+AnastasiiaVashchuk@users.noreply.github.com> Date: Thu, 20 Jun 2024 13:04:22 +0300 Subject: [PATCH 32/43] feat(api): Add new `l1_committed` block tag (#2282) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Follow-ups: - add `l1_committed` tag to zksync-ethers codebase. - update docs. ## What ❔ - added the new tag `L1Committed` to `BlockNumber` enum. - added a new query to db to fetch l2 block, that was included into committed on L1 `l1_batch` inside `resolve_block_id ` dal method. - added unit test to check new feature `resolving_l1_committed_block_id `. - updated integration test. - refactored `dal`'s tests - created `create_l1_batch_header` for tests to avoid duplication. ## Why ❔ Chainlink needs this feature. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- Cargo.lock | 11 ++- ...ae1cee8eeca1c74529adee78157dcf8930c44.json | 20 ++++ core/lib/dal/src/blocks_dal.rs | 12 +-- core/lib/dal/src/blocks_web3_dal.rs | 96 ++++++++++++++++++- core/lib/dal/src/helpers.rs | 15 +-- core/lib/dal/src/pruning_dal/tests.rs | 16 +--- core/lib/dal/src/tests/mod.rs | 13 ++- core/lib/types/src/api/mod.rs | 4 + core/node/api_server/src/web3/metrics.rs | 2 + .../ts-integration/tests/api/web3.test.ts | 2 + 10 files changed, 151 insertions(+), 40 deletions(-) create mode 100644 core/lib/dal/.sqlx/query-01cc22f19b61145b0dbed96ec84ae1cee8eeca1c74529adee78157dcf8930c44.json diff --git a/Cargo.lock b/Cargo.lock index b013517e0cc2..3582fbe51319 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1509,15 +1509,16 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.3" +version = "4.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", + "platforms", "rustc_version", "subtle", "zeroize", @@ -4386,6 +4387,12 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +[[package]] +name = "platforms" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" + [[package]] name = "plotters" version = "0.3.5" diff --git a/core/lib/dal/.sqlx/query-01cc22f19b61145b0dbed96ec84ae1cee8eeca1c74529adee78157dcf8930c44.json b/core/lib/dal/.sqlx/query-01cc22f19b61145b0dbed96ec84ae1cee8eeca1c74529adee78157dcf8930c44.json new file mode 100644 index 000000000000..d8af3cae95b9 --- /dev/null +++ b/core/lib/dal/.sqlx/query-01cc22f19b61145b0dbed96ec84ae1cee8eeca1c74529adee78157dcf8930c44.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT COALESCE(\n (\n SELECT MAX(number) FROM miniblocks\n WHERE l1_batch_number = (\n SELECT number FROM l1_batches\n JOIN eth_txs ON\n l1_batches.eth_commit_tx_id = eth_txs.id\n WHERE\n eth_txs.confirmed_eth_tx_history_id IS NOT NULL\n ORDER BY number DESC LIMIT 1\n )\n ),\n 0\n ) AS number\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "number", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "01cc22f19b61145b0dbed96ec84ae1cee8eeca1c74529adee78157dcf8930c44" +} diff --git a/core/lib/dal/src/blocks_dal.rs b/core/lib/dal/src/blocks_dal.rs index 94d3b3372d93..2e59c2db50e4 100644 --- a/core/lib/dal/src/blocks_dal.rs +++ b/core/lib/dal/src/blocks_dal.rs @@ -2366,7 +2366,7 @@ mod tests { }; use super::*; - use crate::{ConnectionPool, Core, CoreDal}; + use crate::{tests::create_l1_batch_header, ConnectionPool, Core, CoreDal}; async fn save_mock_eth_tx(action_type: AggregatedActionType, conn: &mut Connection<'_, Core>) { conn.eth_sender_dal() @@ -2376,15 +2376,7 @@ mod tests { } fn mock_l1_batch_header() -> L1BatchHeader { - let mut header = L1BatchHeader::new( - L1BatchNumber(1), - 100, - BaseSystemContractsHashes { - bootloader: H256::repeat_byte(1), - default_aa: H256::repeat_byte(42), - }, - ProtocolVersionId::latest(), - ); + let mut header = create_l1_batch_header(1); header.l1_tx_count = 3; header.l2_tx_count = 5; header.l2_to_l1_logs.push(UserL2ToL1Log(L2ToL1Log { diff --git a/core/lib/dal/src/blocks_web3_dal.rs b/core/lib/dal/src/blocks_web3_dal.rs index 1c7f912728cc..b1637d2124b5 100644 --- a/core/lib/dal/src/blocks_web3_dal.rs +++ b/core/lib/dal/src/blocks_web3_dal.rs @@ -271,6 +271,24 @@ impl BlocksWeb3Dal<'_, '_> { api::BlockId::Number(api::BlockNumber::Latest | api::BlockNumber::Committed) => ( "SELECT MAX(number) AS number FROM miniblocks"; ), + api::BlockId::Number(api::BlockNumber::L1Committed) => ( + " + SELECT COALESCE( + ( + SELECT MAX(number) FROM miniblocks + WHERE l1_batch_number = ( + SELECT number FROM l1_batches + JOIN eth_txs ON + l1_batches.eth_commit_tx_id = eth_txs.id + WHERE + eth_txs.confirmed_eth_tx_history_id IS NOT NULL + ORDER BY number DESC LIMIT 1 + ) + ), + 0 + ) AS number + "; + ), api::BlockId::Number(api::BlockNumber::Finalized) => ( " SELECT COALESCE( @@ -733,6 +751,7 @@ impl BlocksWeb3Dal<'_, '_> { #[cfg(test)] mod tests { use zksync_types::{ + aggregated_operations::AggregatedActionType, block::{L2BlockHasher, L2BlockHeader}, fee::TransactionExecutionMetrics, Address, L2BlockNumber, ProtocolVersion, ProtocolVersionId, @@ -741,8 +760,8 @@ mod tests { use super::*; use crate::{ tests::{ - create_l2_block_header, create_snapshot_recovery, mock_execution_result, - mock_l2_transaction, + create_l1_batch_header, create_l2_block_header, create_snapshot_recovery, + mock_execution_result, mock_l2_transaction, }, ConnectionPool, Core, CoreDal, }; @@ -902,6 +921,79 @@ mod tests { assert_eq!(l2_block_number, Some(L2BlockNumber(43))); } + #[tokio::test] + async fn resolving_l1_committed_block_id() { + let connection_pool = ConnectionPool::::test_pool().await; + let mut conn = connection_pool.connection().await.unwrap(); + conn.protocol_versions_dal() + .save_protocol_version_with_tx(&ProtocolVersion::default()) + .await + .unwrap(); + + let l2_block_header = create_l2_block_header(1); + conn.blocks_dal() + .insert_l2_block(&l2_block_header) + .await + .unwrap(); + + let l1_batch_header = create_l1_batch_header(0); + + conn.blocks_dal() + .insert_mock_l1_batch(&l1_batch_header) + .await + .unwrap(); + conn.blocks_dal() + .mark_l2_blocks_as_executed_in_l1_batch(l1_batch_header.number) + .await + .unwrap(); + + let resolved_l2_block_number = conn + .blocks_web3_dal() + .resolve_block_id(api::BlockId::Number(api::BlockNumber::L1Committed)) + .await + .unwrap(); + assert_eq!(resolved_l2_block_number, Some(L2BlockNumber(0))); + + let mocked_commit_eth_tx = conn + .eth_sender_dal() + .save_eth_tx( + 0, + vec![], + AggregatedActionType::Commit, + Address::default(), + 0, + None, + None, + ) + .await + .unwrap(); + let tx_hash = H256::random(); + conn.eth_sender_dal() + .insert_tx_history(mocked_commit_eth_tx.id, 0, 0, None, tx_hash, &[], 0) + .await + .unwrap(); + conn.eth_sender_dal() + .confirm_tx(tx_hash, U256::zero()) + .await + .unwrap(); + conn.blocks_dal() + .set_eth_tx_id( + l1_batch_header.number..=l1_batch_header.number, + mocked_commit_eth_tx.id, + AggregatedActionType::Commit, + ) + .await + .unwrap(); + + let resolved_l2_block_number = conn + .blocks_web3_dal() + .resolve_block_id(api::BlockId::Number(api::BlockNumber::L1Committed)) + .await + .unwrap(); + + assert_eq!(resolved_l2_block_number, Some(l2_block_header.number)); + } + #[tokio::test] async fn resolving_block_by_hash() { let connection_pool = ConnectionPool::::test_pool().await; diff --git a/core/lib/dal/src/helpers.rs b/core/lib/dal/src/helpers.rs index e8e11f1cc5f3..65e9161bd047 100644 --- a/core/lib/dal/src/helpers.rs +++ b/core/lib/dal/src/helpers.rs @@ -40,11 +40,10 @@ pub async fn wait_for_l1_batch( #[cfg(test)] mod tests { - use zksync_contracts::BaseSystemContractsHashes; - use zksync_types::{block::L1BatchHeader, ProtocolVersion, ProtocolVersionId, H256}; + use zksync_types::ProtocolVersion; use super::*; - use crate::{ConnectionPool, Core, CoreDal}; + use crate::{tests::create_l1_batch_header, ConnectionPool, Core, CoreDal}; #[tokio::test] async fn waiting_for_l1_batch_success() { @@ -59,15 +58,7 @@ mod tests { .save_protocol_version_with_tx(&ProtocolVersion::default()) .await .unwrap(); - let header = L1BatchHeader::new( - L1BatchNumber(0), - 100, - BaseSystemContractsHashes { - bootloader: H256::repeat_byte(1), - default_aa: H256::repeat_byte(42), - }, - ProtocolVersionId::latest(), - ); + let header = create_l1_batch_header(0); conn.blocks_dal() .insert_mock_l1_batch(&header) .await diff --git a/core/lib/dal/src/pruning_dal/tests.rs b/core/lib/dal/src/pruning_dal/tests.rs index 61b5766b93e7..1c3b1edcbd42 100644 --- a/core/lib/dal/src/pruning_dal/tests.rs +++ b/core/lib/dal/src/pruning_dal/tests.rs @@ -1,9 +1,7 @@ use std::ops; -use zksync_contracts::BaseSystemContractsHashes; use zksync_db_connection::connection::Connection; use zksync_types::{ - block::L1BatchHeader, fee::TransactionExecutionMetrics, l2_to_l1_log::{L2ToL1Log, UserL2ToL1Log}, tx::IncludedTxLocation, @@ -15,8 +13,8 @@ use super::*; use crate::{ storage_logs_dal::DbStorageLog, tests::{ - create_l2_block_header, mock_execution_result, mock_l2_to_l1_log, mock_l2_transaction, - mock_vm_event, + create_l1_batch_header, create_l2_block_header, mock_execution_result, mock_l2_to_l1_log, + mock_l2_transaction, mock_vm_event, }, ConnectionPool, Core, CoreDal, }; @@ -89,15 +87,7 @@ async fn insert_events(conn: &mut Connection<'_, Core>, l2_block_number: L2Block } async fn insert_l1_batch(conn: &mut Connection<'_, Core>, l1_batch_number: L1BatchNumber) { - let mut header = L1BatchHeader::new( - l1_batch_number, - 100, - BaseSystemContractsHashes { - bootloader: H256::repeat_byte(1), - default_aa: H256::repeat_byte(42), - }, - ProtocolVersionId::latest(), - ); + let mut header = create_l1_batch_header(*l1_batch_number); header.l1_tx_count = 3; header.l2_tx_count = 5; header.l2_to_l1_logs.push(UserL2ToL1Log(L2ToL1Log { diff --git a/core/lib/dal/src/tests/mod.rs b/core/lib/dal/src/tests/mod.rs index c4dab1246552..d6ffde594323 100644 --- a/core/lib/dal/src/tests/mod.rs +++ b/core/lib/dal/src/tests/mod.rs @@ -3,7 +3,7 @@ use std::time::Duration; use zksync_contracts::BaseSystemContractsHashes; use zksync_db_connection::connection_pool::ConnectionPool; use zksync_types::{ - block::{L2BlockHasher, L2BlockHeader}, + block::{L1BatchHeader, L2BlockHasher, L2BlockHeader}, fee::{Fee, TransactionExecutionMetrics}, fee_model::BatchFeeInput, helpers::unix_timestamp_ms, @@ -50,6 +50,17 @@ pub(crate) fn create_l2_block_header(number: u32) -> L2BlockHeader { gas_limit: 0, } } +pub(crate) fn create_l1_batch_header(number: u32) -> L1BatchHeader { + L1BatchHeader::new( + L1BatchNumber(number), + 100, + BaseSystemContractsHashes { + bootloader: H256::repeat_byte(1), + default_aa: H256::repeat_byte(42), + }, + ProtocolVersionId::latest(), + ) +} pub(crate) fn mock_l2_transaction() -> L2Tx { let fee = Fee { diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index ce21a754c7aa..0617f47268aa 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -27,6 +27,8 @@ pub enum BlockNumber { Finalized, /// Latest sealed block Latest, + /// Last block that was committed on L1 + L1Committed, /// Earliest block (genesis) Earliest, /// Latest block (may be the block that is currently open). @@ -51,6 +53,7 @@ impl Serialize for BlockNumber { BlockNumber::Committed => serializer.serialize_str("committed"), BlockNumber::Finalized => serializer.serialize_str("finalized"), BlockNumber::Latest => serializer.serialize_str("latest"), + BlockNumber::L1Committed => serializer.serialize_str("l1_committed"), BlockNumber::Earliest => serializer.serialize_str("earliest"), BlockNumber::Pending => serializer.serialize_str("pending"), } @@ -73,6 +76,7 @@ impl<'de> Deserialize<'de> for BlockNumber { "committed" => BlockNumber::Committed, "finalized" => BlockNumber::Finalized, "latest" => BlockNumber::Latest, + "l1_committed" => BlockNumber::L1Committed, "earliest" => BlockNumber::Earliest, "pending" => BlockNumber::Pending, num => { diff --git a/core/node/api_server/src/web3/metrics.rs b/core/node/api_server/src/web3/metrics.rs index af6e1bf63ad8..9d8cbf813b03 100644 --- a/core/node/api_server/src/web3/metrics.rs +++ b/core/node/api_server/src/web3/metrics.rs @@ -102,6 +102,7 @@ enum BlockIdLabel { Committed, Finalized, Latest, + L1Committed, Earliest, Pending, Number, @@ -139,6 +140,7 @@ impl From<&MethodMetadata> for MethodLabels { api::BlockId::Number(api::BlockNumber::Committed) => BlockIdLabel::Committed, api::BlockId::Number(api::BlockNumber::Finalized) => BlockIdLabel::Finalized, api::BlockId::Number(api::BlockNumber::Latest) => BlockIdLabel::Latest, + api::BlockId::Number(api::BlockNumber::L1Committed) => BlockIdLabel::L1Committed, api::BlockId::Number(api::BlockNumber::Earliest) => BlockIdLabel::Earliest, api::BlockId::Number(api::BlockNumber::Pending) => BlockIdLabel::Pending, }); diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index 09f78ce75059..3eb4afb3977e 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -690,6 +690,8 @@ describe('web3 API compatibility tests', () => { expect(+finalizedBlock.number!).toEqual(expect.any(Number)); const latestBlock = await alice.provider.send('eth_getBlockByNumber', ['latest', true]); expect(+latestBlock.number!).toEqual(expect.any(Number)); + const l1CommittedBlock = await alice.provider.send('eth_getBlockByNumber', ['l1_committed', true]); + expect(+l1CommittedBlock.number!).toEqual(expect.any(Number)); const pendingBlock = await alice.provider.send('eth_getBlockByNumber', ['pending', true]); expect(pendingBlock).toEqual(null); }); From 7bd8f27e5171f37da3aa1d6c6abb06b9a291fbbf Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 20 Jun 2024 13:58:46 +0300 Subject: [PATCH 33/43] fix(pruning): Check pruning in metadata calculator (#2286) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Adds checks in the case the metadata calculator cannot proceed because of pruning. ## Why ❔ While we don't officially support distributed setup for ENs, it still looks worth it to have intelligent error messages in case metadata calculator is stuck. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- .../metadata_calculator/src/recovery/mod.rs | 4 +- core/node/metadata_calculator/src/tests.rs | 40 +++++++++++++++++++ core/node/metadata_calculator/src/updater.rs | 15 +++++++ 3 files changed, 58 insertions(+), 1 deletion(-) diff --git a/core/node/metadata_calculator/src/recovery/mod.rs b/core/node/metadata_calculator/src/recovery/mod.rs index b4e91bf720ee..4aee14c0c79a 100644 --- a/core/node/metadata_calculator/src/recovery/mod.rs +++ b/core/node/metadata_calculator/src/recovery/mod.rs @@ -279,7 +279,9 @@ impl AsyncTreeRecovery { let actual_root_hash = tree.root_hash().await; anyhow::ensure!( actual_root_hash == snapshot.expected_root_hash, - "Root hash of recovered tree {actual_root_hash:?} differs from expected root hash {:?}", + "Root hash of recovered tree {actual_root_hash:?} differs from expected root hash {:?}. \ + If pruning is enabled and the tree is initialized some time after node recovery, \ + this is caused by snapshot storage logs getting pruned; this setup is currently not supported", snapshot.expected_root_hash ); let tree = tree.finalize().await?; diff --git a/core/node/metadata_calculator/src/tests.rs b/core/node/metadata_calculator/src/tests.rs index fbdfe6cab322..38e1a09d1091 100644 --- a/core/node/metadata_calculator/src/tests.rs +++ b/core/node/metadata_calculator/src/tests.rs @@ -360,6 +360,46 @@ async fn multi_l1_batch_workflow() { } } +#[tokio::test] +async fn error_on_pruned_next_l1_batch() { + let pool = ConnectionPool::::test_pool().await; + let temp_dir = TempDir::new().expect("failed get temporary directory for RocksDB"); + let (calculator, _) = setup_calculator(temp_dir.path(), pool.clone()).await; + reset_db_state(&pool, 1).await; + run_calculator(calculator).await; + + // Add some new blocks to the storage and mock their partial pruning. + let mut storage = pool.connection().await.unwrap(); + let new_logs = gen_storage_logs(100..200, 10); + extend_db_state(&mut storage, new_logs).await; + storage + .pruning_dal() + .soft_prune_batches_range(L1BatchNumber(5), L2BlockNumber(5)) + .await + .unwrap(); + storage + .pruning_dal() + .hard_prune_batches_range(L1BatchNumber(5), L2BlockNumber(5)) + .await + .unwrap(); + // Sanity check: there should be no pruned batch headers. + let next_l1_batch_header = storage + .blocks_dal() + .get_l1_batch_header(L1BatchNumber(2)) + .await + .unwrap(); + assert!(next_l1_batch_header.is_none()); + + let (calculator, _) = setup_calculator(temp_dir.path(), pool.clone()).await; + let (_stop_sender, stop_receiver) = watch::channel(false); + let err = calculator.run(stop_receiver).await.unwrap_err(); + let err = format!("{err:#}"); + assert!( + err.contains("L1 batch #2, next to be processed by the tree, is pruned"), + "{err}" + ); +} + #[tokio::test] async fn running_metadata_calculator_with_additional_blocks() { let pool = ConnectionPool::::test_pool().await; diff --git a/core/node/metadata_calculator/src/updater.rs b/core/node/metadata_calculator/src/updater.rs index 8271865199a8..2056b8315664 100644 --- a/core/node/metadata_calculator/src/updater.rs +++ b/core/node/metadata_calculator/src/updater.rs @@ -103,6 +103,7 @@ impl TreeUpdater { for l1_batch_number in l1_batch_numbers { let l1_batch_number = L1BatchNumber(l1_batch_number); let Some(current_l1_batch_data) = l1_batch_data else { + Self::ensure_not_pruned(storage, l1_batch_number).await?; return Ok(l1_batch_number); }; total_logs += current_l1_batch_data.storage_logs.len(); @@ -167,6 +168,20 @@ impl TreeUpdater { Ok(last_l1_batch_number + 1) } + /// Checks whether the requested L1 batch was pruned. Right now, the tree cannot recover from this situation, + /// so we exit with an error if this happens. + async fn ensure_not_pruned( + storage: &mut Connection<'_, Core>, + l1_batch_number: L1BatchNumber, + ) -> anyhow::Result<()> { + let pruning_info = storage.pruning_dal().get_pruning_info().await?; + anyhow::ensure!( + Some(l1_batch_number) > pruning_info.last_soft_pruned_l1_batch, + "L1 batch #{l1_batch_number}, next to be processed by the tree, is pruned; the tree cannot continue operating" + ); + Ok(()) + } + async fn step( &mut self, mut storage: Connection<'_, Core>, From 3f88b8d10351b56f68f9710bb8c145e3f6655da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Thu, 20 Jun 2024 15:56:34 +0200 Subject: [PATCH 34/43] fix(ci): missing run-retried for rust installation (#2277) Signed-off-by: tomg10 --- .github/workflows/build-contract-verifier-template.yml | 2 +- .github/workflows/build-core-template.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-contract-verifier-template.yml b/.github/workflows/build-contract-verifier-template.yml index 3068b3414774..2b24801d065f 100644 --- a/.github/workflows/build-contract-verifier-template.yml +++ b/.github/workflows/build-contract-verifier-template.yml @@ -138,7 +138,7 @@ jobs: COMPONENT: ${{ matrix.components }} PLATFORM: ${{ matrix.platforms }} run: | - ci_run rustup default nightly-2024-05-07 + ci_run run_retried rustup default nightly-2024-05-07 platform=$(echo $PLATFORM | tr '/' '-') ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT - name: Show sccache stats diff --git a/.github/workflows/build-core-template.yml b/.github/workflows/build-core-template.yml index 49b619a7f944..4ead6cb746dd 100644 --- a/.github/workflows/build-core-template.yml +++ b/.github/workflows/build-core-template.yml @@ -147,7 +147,7 @@ jobs: COMPONENT: ${{ matrix.components }} PLATFORM: ${{ matrix.platforms }} run: | - ci_run rustup default nightly-2024-05-07 + ci_run run_retried rustup default nightly-2024-05-07 platform=$(echo $PLATFORM | tr '/' '-') ci_run zk docker $DOCKER_ACTION --custom-tag=${IMAGE_TAG_SUFFIX} --platform=${PLATFORM} $COMPONENT - name: Show sccache stats From dd2c941e94b5fde9064d9ff4394a00a4b0707fbf Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Thu, 20 Jun 2024 17:00:42 +0300 Subject: [PATCH 35/43] =?UTF-8?q?refactor(db):=20Improve=20storage=20logs?= =?UTF-8?q?=20pruning=20query=20=E2=80=93=20add=20block=20condition=20(#22?= =?UTF-8?q?85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Adds a block condition for the storage logs pruning query. ## Why ❔ This should improve the query plan. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- ...685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64.json} | 4 ++-- core/lib/dal/src/pruning_dal/mod.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) rename core/lib/dal/.sqlx/{query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json => query-ead71ae66fe4685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64.json} (74%) diff --git a/core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json b/core/lib/dal/.sqlx/query-ead71ae66fe4685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64.json similarity index 74% rename from core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json rename to core/lib/dal/.sqlx/query-ead71ae66fe4685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64.json index 93d1966f370c..02cd6733e811 100644 --- a/core/lib/dal/.sqlx/query-6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25.json +++ b/core/lib/dal/.sqlx/query-ead71ae66fe4685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n WITH\n new_logs AS MATERIALIZED (\n SELECT DISTINCT\n ON (hashed_key) hashed_key,\n miniblock_number,\n operation_number\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n ORDER BY\n hashed_key,\n miniblock_number DESC,\n operation_number DESC\n )\n DELETE FROM storage_logs USING new_logs\n WHERE\n storage_logs.hashed_key = new_logs.hashed_key\n AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number)\n ", + "query": "\n WITH\n new_logs AS MATERIALIZED (\n SELECT DISTINCT\n ON (hashed_key) hashed_key,\n miniblock_number,\n operation_number\n FROM\n storage_logs\n WHERE\n miniblock_number BETWEEN $1 AND $2\n ORDER BY\n hashed_key,\n miniblock_number DESC,\n operation_number DESC\n )\n DELETE FROM storage_logs USING new_logs\n WHERE\n storage_logs.hashed_key = new_logs.hashed_key\n AND storage_logs.miniblock_number <= $2\n AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number)\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "6ad9adcbd60483148983a495d0e9b5c09854efaa4c0a35466b138587dce03f25" + "hash": "ead71ae66fe4685132c03a973612fe98364aa684180dd6fbf540bb0b68d96a64" } diff --git a/core/lib/dal/src/pruning_dal/mod.rs b/core/lib/dal/src/pruning_dal/mod.rs index 7f30af034e2b..0d1584ebba36 100644 --- a/core/lib/dal/src/pruning_dal/mod.rs +++ b/core/lib/dal/src/pruning_dal/mod.rs @@ -336,6 +336,7 @@ impl PruningDal<'_, '_> { DELETE FROM storage_logs USING new_logs WHERE storage_logs.hashed_key = new_logs.hashed_key + AND storage_logs.miniblock_number <= $2 AND (storage_logs.miniblock_number, storage_logs.operation_number) < (new_logs.miniblock_number, new_logs.operation_number) "#, i64::from(l2_blocks_to_prune.start().0), From 619a525bc8f1098297259ddb296b4b5dee223944 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Thu, 20 Jun 2024 18:38:37 +0200 Subject: [PATCH 36/43] feat(docs): Pruning and Snapshots recovery basic docs (#2265) Signed-off-by: tomg10 --- core/bin/external_node/src/config/mod.rs | 2 +- .../external-node/07_snapshots_recovery.md | 30 ++++++++++++++ docs/guides/external-node/08_pruning.md | 40 +++++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 docs/guides/external-node/07_snapshots_recovery.md create mode 100644 docs/guides/external-node/08_pruning.md diff --git a/core/bin/external_node/src/config/mod.rs b/core/bin/external_node/src/config/mod.rs index 9cd6a758a25c..b47ae3f8886e 100644 --- a/core/bin/external_node/src/config/mod.rs +++ b/core/bin/external_node/src/config/mod.rs @@ -541,7 +541,7 @@ impl OptionalENConfig { } fn default_pruning_data_retention_sec() -> u64 { - 3_600 // 1 hour + 3_600 * 24 * 7 // 7 days } fn from_env() -> anyhow::Result { diff --git a/docs/guides/external-node/07_snapshots_recovery.md b/docs/guides/external-node/07_snapshots_recovery.md new file mode 100644 index 000000000000..94d279e358de --- /dev/null +++ b/docs/guides/external-node/07_snapshots_recovery.md @@ -0,0 +1,30 @@ +# Snapshots Recovery + +Instead of starting node using DB snapshots, it's possible to configure them to start from a protocol-level snapshots. +This process is much faster and requires way less storage. Postgres database of a mainnet node recovered from a snapshot +is only about 300GB. Without [_pruning_](08_pruning.md) enabled, the state will continuously grow about 15GB per day. + +> [!NOTE] +> +> Nodes recovered from snapshot don't have any historical data from before the recovery! + +## Configuration + +To enable snapshots-recovery on mainnet, you need to set environment variables: + +```yaml +EN_SNAPSHOTS_RECOVERY_ENABLED: 'true' +EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL: 'zksync-era-mainnet-external-node-snapshots' +EN_SNAPSHOTS_OBJECT_STORE_MODE: 'GCSAnonymousReadOnly' +``` + +For sepolia testnet, use: + +```yaml +EN_SNAPSHOTS_RECOVERY_ENABLED: 'true' +EN_SNAPSHOTS_OBJECT_STORE_BUCKET_BASE_URL: 'zksync-era-boojnet-external-node-snapshots' +EN_SNAPSHOTS_OBJECT_STORE_MODE: 'GCSAnonymousReadOnly' +``` + +For a working examples of a fully configured Nodes recovering from snapshots, see +[_docker compose examples_](docker-compose-examples) directory and [_Quick Start_](00_quick_start.md) diff --git a/docs/guides/external-node/08_pruning.md b/docs/guides/external-node/08_pruning.md new file mode 100644 index 000000000000..c7f834214ae7 --- /dev/null +++ b/docs/guides/external-node/08_pruning.md @@ -0,0 +1,40 @@ +# Pruning + +It is possible to configure ZKsync Node to periodically remove all data from batches older than a configurable +threshold. Data is pruned both from Postgres and from tree (RocksDB). + +> [!NOTE] +> +> If you need a node with data retention period of up to a few days, please set up a node from a +> [_snapshot_](07_snapshots_recovery.md) and wait for it to have enough data. Pruning an archival node can take +> unpractical amount of time. In the future we will be offering pre-pruned DB snapshots with a few months of data. + +## Configuration + +You can enable pruning by setting the environment variable + +```yaml +EN_PRUNING_ENABLED: 'true' +``` + +By default, it will keep history for 7 days. You can configure retention period using: + +```yaml +EN_PRUNING_DATA_RETENTION_SEC: '259200' # 3 days +``` + +The data retention can be set to any value, but for mainnet values under 21h will be ignored as the batch can only be +pruned as soon as it has been executed on Ethereum. + +## Storage requirements for pruned nodes + +The storage requirements depend on how long you configure to retain the data, but are roughly: + +- **40GB + ~5GB/day of retained data** of disk space needed on machine that runs the node +- **300GB + ~15GB/day of retained data** of disk space for Postgres + +> [!NOTE] +> +> When pruning an existing archival node, Postgres will be unable to reclaim disk space automatically, to reclaim disk +> space, you need to manually run VACUUM FULL, which requires an ACCESS EXCLUSIVE lock, you can read more about it in +> [_postgres docs_](https://www.postgresql.org/docs/current/sql-vacuum.html) From f4aff94d714fb9a8eaff3d6735b9fc1033b09e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Grze=C5=9Bkiewicz?= Date: Thu, 20 Jun 2024 19:38:28 +0200 Subject: [PATCH 37/43] feat(ci): remove spellcheck (#2243) Signed-off-by: tomg10 Co-authored-by: pompon0 --- .github/pull_request_template.md | 1 - .github/workflows/check-spelling.yml | 42 -- CONTRIBUTING.md | 4 +- checks-config/cspell.json | 47 -- checks-config/era.cfg | 69 -- checks-config/era.dic | 984 --------------------------- checks-config/links.json | 32 - docker/zk-environment/Dockerfile | 3 +- docs/guides/development.md | 50 -- infrastructure/zk/package.json | 1 - infrastructure/zk/src/index.ts | 2 - infrastructure/zk/src/spellcheck.ts | 44 -- 12 files changed, 2 insertions(+), 1277 deletions(-) delete mode 100644 .github/workflows/check-spelling.yml delete mode 100644 checks-config/cspell.json delete mode 100644 checks-config/era.cfg delete mode 100644 checks-config/era.dic delete mode 100644 checks-config/links.json delete mode 100644 infrastructure/zk/src/spellcheck.ts diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 764b85baccaf..dba6efd2fdff 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -18,4 +18,3 @@ - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. -- [ ] Spellcheck has been run via `zk spellcheck`. diff --git a/.github/workflows/check-spelling.yml b/.github/workflows/check-spelling.yml deleted file mode 100644 index 8ffa29c1ea9b..000000000000 --- a/.github/workflows/check-spelling.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Check Spelling - -on: - push: - branches: - - main - pull_request: - merge_group: - -env: - CARGO_TERM_COLOR: always - -jobs: - spellcheck: - runs-on: [matterlabs-ci-runner] - steps: - - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 - with: - submodules: "recursive" - - name: Use Node.js - uses: actions/setup-node@v3 - with: - node-version: 18 - - - name: Setup environment - run: | - echo ZKSYNC_HOME=$(pwd) >> $GITHUB_ENV - echo $(pwd)/bin >> $GITHUB_PATH - echo IN_DOCKER=1 >> .env - - - name: Start services - run: | - run_retried docker compose pull zk - docker compose up -d zk - - - name: Build zk - run: | - ci_run zk - - - name: Run spellcheck - run: | - ci_run zk spellcheck diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2676289d0f3a..0791a311fed3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,8 +52,6 @@ Be polite and respectful. **Q**: I have a small contribution that's not getting traction/being merged? **A**: Due to capacity, contributions that are simple renames of variables or stylistic/minor text improvements, one-off -typo fix will not be merged. If you do find any typos or grammar errors, the preferred avenue is to improve the existing -spellchecker. Given you have no technical prowess to do so, please create an issue. Please note that issues will be -resolved on a best effort basis. +typo fix will not be merged. ### Thank you diff --git a/checks-config/cspell.json b/checks-config/cspell.json deleted file mode 100644 index bafb5e036d04..000000000000 --- a/checks-config/cspell.json +++ /dev/null @@ -1,47 +0,0 @@ -{ - "language": "en", - "ignorePaths": [ - "**/CHANGELOG.md", - "**/node_modules/**", - ".github/**", - ".firebase/**", - ".yarn/**", - "dist/**", - "**/contracts/**", - "**/target/**" - ], - "dictionaries": [ - "typescript", - "cpp", - "npm", - "filetypes", - "cpp", - "en_GB", - "en_US", - "node", - "bash", - "fonts", - "npm", - "cryptocurrencies", - "companies", - "rust", - "html", - "css", - "entities", - "softwareTerms", - "misc", - "fullstack", - "softwareTerms", - "zksync", - "nuxt", - "viem" - ], - "dictionaryDefinitions": [ - { - "name": "zksync", - "addWords": true, - "path": "./era.dic" - } - ], - "allowCompoundWords": true - } \ No newline at end of file diff --git a/checks-config/era.cfg b/checks-config/era.cfg deleted file mode 100644 index c8a6baba820a..000000000000 --- a/checks-config/era.cfg +++ /dev/null @@ -1,69 +0,0 @@ -# Project settings where a Cargo.toml exists and is passed -# ${CARGO_MANIFEST_DIR}/.config/spellcheck.toml - -# Also take into account developer comments -dev_comments = true - -# Skip the README.md file as defined in the cargo manifest -skip_readme = false - -[Hunspell] -# lang and name of `.dic` file -lang = "en_US" -# OS specific additives -# Linux: [ /usr/share/myspell ] -# Windows: [] -# macOS [ /home/alice/Libraries/hunspell, /Libraries/hunspell ] - -# Additional search paths, which take precedence over the default -# os specific search dirs, searched in order, defaults last -search_dirs = ["."] - -# Adds additional dictionaries, can be specified as -# absolute paths or relative in the search dirs (in this order). -# Relative paths are resolved relative to the configuration file -# which is used. -# Refer to `man 5 hunspell` -# or https://www.systutorials.com/docs/linux/man/4-hunspell/#lbAE -# on how to define a custom dictionary file. -extra_dictionaries = ["era.dic"] - -# If set to `true`, the OS specific default search paths -# are skipped and only explicitly specified ones are used. -skip_os_lookups = false - -# Use the builtin dictionaries if none were found in -# in the configured lookup paths. -# Usually combined with `skip_os_lookups=true` -# to enforce the `builtin` usage for consistent -# results across distributions and CI runs. -# Setting this will still use the dictionaries -# specified in `extra_dictionaries = [..]` -# for topic specific lingo. -use_builtin = true - - -[Hunspell.quirks] -# Transforms words that are provided by the tokenizer -# into word fragments based on the capture groups which are to -# be checked. -# If no capture groups are present, the matched word is whitelisted. -transform_regex = ["^'([^\\s])'$", "^[0-9]+x$"] -# Accepts `alphabeta` variants if the checker provides a replacement suggestion -# of `alpha-beta`. -allow_concatenation = true -# And the counterpart, which accepts words with dashes, when the suggestion has -# recommendations without the dashes. This is less common. -allow_dashed = false - -[NlpRules] -# Allows the user to override the default included -# exports of LanguageTool, with other custom -# languages - -# override_rules = "/path/to/rules_binencoded.bin" -# override_tokenizer = "/path/to/tokenizer_binencoded.bin" - -[Reflow] -# Reflows doc comments to adhere to a given maximum line width limit. -max_line_length = 80 diff --git a/checks-config/era.dic b/checks-config/era.dic deleted file mode 100644 index 3f4c8fc8fa4f..000000000000 --- a/checks-config/era.dic +++ /dev/null @@ -1,984 +0,0 @@ -42 -<= -=> -== --> -<- -+ -- -* -\ -= -/ -|| -< -> -% -^ -0x00 -0x01 -0x02 -0x20 -~10x -u32 -u64 -u8 -1B -H256 -10e18 -10^9 -2^32 -2^128 -2^24 -10^32 -10^* -2^16 -2^64 -10^8 -U256 -12.5% -5% -10% -20% -*% -90% -1% -f64 -k -M -kb -Gbps -50M -2M -130µs -– -18kb -128kb -10k -100k -120k -800k -24k -500k -50k -52k -260k -120kb -18kb -12GB -20GB -500B -100M -~100us -10ms -1_000ms -1us -~100 -gwei - -ABI -vlog -const -L2 -L2s -L1 -json -l1 -l2 -G1 -G2 -SystemConfig -TODO -se -ZKSYNC_HOME -MultiVMTracer -vm_virtual_blocks -eth_node -EthCall -BaseSystemContracts -eth_calls -refactor -WS -env -url -GasAdjuster -base_fee -base_fee_per_gas -ERC20 -Finalizer -Backoff -middleware -parallelization -precompute -precomputed -Postgres -parallelized -parallelize -job_id -API -APIs -async -pointwise -observability -atomics -integrations -stdout -GCS -websocket -struct -struct's -localhost -TOML -config -finalizer -boolean -prover -timestamp -H160 -ZKsync -AccessList -miniblock -member₁ -member₂ -memberₙ -merkle -eth -Ethereum -deployer -designator -designators -RPC -tx -txs -subtrees -subtree -unfinalizable -meterer -Timedout -bootloader -bootloader's -testkit -Sepolia -Goerli -miniblock -miniblocks -MempoolIO -mempool -latencies -OracleTools -StorageOracle -zk_evm -zkEVM -src -utils -ptr -recurse -RefCell -Rc -StorageView -VM_HOOK_POSITION -VM_HOOKS_PARAMS_COUNT -PAYMASTER_CONTEXT_SLOTS -PrecompilerProcessor -MAX_POSTOP_SLOTS -postOp -type -opcode -KnownCodesStorage -param -HistoryDisabled -HistoryEnabled -sorted_timestamps -known_bytecodes -returndata -namespaces -natively -StateDiffRecord -BYTES_PER_ENUMERATION_INDEX -derived_key -prefill -reorg -precompile -Init -init -enqueued -stage2 -testnets -ethCalls -generable -Serde -tokenize -EOAs -zeroized -cardinality - -// ZKsync-related words -matterlabs -zkweb -zksync -blockchain -zkscan -zkscrypto -PubSub -loadtest -BigUint -radix -state_keeper -MIN_PAYMASTER_BALANCE -PrometheusCollector -RetryCollector -ScriptCollector -MetricsCollector -OperationResultsCollector -ReportCollector -filesystem -hasher -Hasher -grafana -prometheus -serializer -serializable -deserializer -Deserializes -deserializes -serializing -deserializing -deserialization -configs -operation_number -hashed_key -deduplication -mutexes -mutex -Blake2s -Blake2 -web3 -Testnets -miniblock_number -hashed_key -tuples -\x19Ethereum -libzkscrypto -EOA -MultiVM -nonces -fri -rollup -pubkey -JSON -keccak256 -pubdata -timestamps -keccak -musig -len -calldata -DApp -metadata -boojum -deps -Precalculated -precalculated -WASM -DefaultPrecompilesProcessor -LSB -DDoS -refactored -tuple -HistoryMode -vm -VM -VMs -VM's -MSB -Enum -PublishProof -jsrpc -backends -ethsig -ethop -decentralization -rollups -zkrollup -unencrypted -permissionless -trustlessness -IERC -Schnorr -MuSig -Merkle -decentralised -mainchain -offchain -processed -zcli -blockchains -sidechain -sidechains -tokenomics -validator -validator's -validator -validators -Validators -CHAINID -PREVRANDAO -ECDSA -EIP712 -EIP1559 -EIPs -eth_estimateGas -eth_call -versa -blake2 -AR16MT -Preimages -EN's -SystemContext -StorageOracle -intrinsics -chunked -chunking -deadbeef01 -deadbeef0 -deadbeef -unsynced -computable -DevEx -Workspace -NFT -preimage -subcalls -hashmaps -monotonicity -subquery -RPCs -programmatically -stdin -stderr -Linter -SmallRng -ZkPorter -StateDiffs -HashMaps -encodings -CTPOP -decommitter -Decommitter -Decommitments -Decommitment -decommitment -decommitments -Decommit -decommit -decommits -DecommiterOracle -DecommitmentProcessor -decommitted -decommit -decommitting -Demuxer -demultiplex -recid -inversed -plux -Binop -Arithmetization -arithmetization -nocapture -Plonky -permissioned -mathbb -Invb -REDC -iszero -skept -ECADD -ECMUL -preds -inttoptr -syncvm -nasm -rodata -ISZERO -JUMPI -ethir -ptrtoint -lshr -getu -zext -noprofile -umin -cccond -ccret -prodm -prodl -prodeh -prodh -interm -signv -ashr -noalias -immediates -prode -StorageBatchInfo -CommitBatchInfo -IExecutor -SetChainId -setChainId -SetChainIdUpgrade -state_transition_manager_contract -prunable -bytea - -// Names -Vyper -stimate -samount -Stichting -Kingsfordweg -RSIN -ABDK -Alef -Zcon -Paypal -Numio -MLTT -USDCs -dapi -validiums -validium -Validium -sharded -pepe -Arweave -Streamr -dutterbutter -NixOS -CLI -SQLx -Rustup -nextest -NTFS -toolchains -toolchain -IDE -M1 -M2 -MacOS -OpenSSL -Xcode -LLVM -nvm -LTS -logout -WSL -orchestrator -TypeScript -Cryptographical -cryptographical -microservices -Executables -subcomponents -v2 -v1 -rmSync -SSL -setup_2^26 -uncomment -toml -GCP -dev -workspace -subcommand -Kubernetes -Etherscan -cryptographic -hashers -MacBook -DDR5 -~ - -// Used libraries -numberish -arrayify -hexlify -markdownlint -ethersproject -nomicfoundation -nomiclabs -Consensys -zkforge -zkcast -Eigen -IPFS - -// Used programming language words -printf -charsets -println -fatalf -allowfullscreen -inttypes -zbin -Panicf -Deri -DERI -Furucombo -kwargs -scaleb -isinstance -RocksDB -mload -secp -porco -rosso -insize -MLOAD -sload -sload -uadd -nocallback -nosync -swrite -Devs -insta -NFTF -yaml - -// ETC -gitter -signup -signups -precompiled -checkmark -Vitalik -Buterin -roadmap -majeure -conveniens -reimplementing -subsecond -supermajority -gemeente -unauthorised -Ethereum's -SDKs -EVM's -EVM -Göerli -ETHUSDC -USDCUSD -ETHUS -USDCUS -ETHUSD -Arbitrum -Adamantium -Immunefi -Winternitz -ewasm -Evmla -UUPS -Uups -TLDR -BLAKE2s -bytes32 -enumeration_index -backend -enum -num_initial -to_check_storage -source_storage -prepend -deduplicated -user_l2_to_l1_logs -L1Messeger -params -provers -zk -substring -reverter -wei -deduplicate -testnet -mainnet -performant -opcodes -USDC -USD -DBs -unexecutable -RLP -DAL -ZKsync's -l2_to_l1 -PoW -coinbase -FIXME -ASC -DESC -versioning -initializer -refactoring -prefetch -unformatted - -// crypto events -Edcon - -// Famous crypto people -Gluchowski -Vitalik's -Buterin's -multisignature -onchain -convertion -Keyhash -Armeabi -scijava -gluk -@Deniallugo's -emilluta - -// Programming related words -backfill -bytecode -bytecodes -impl -subrange -timeframe -leaf_count -mkdir -librocksdb -zksolc -zksyncrobot -precompiles -vyper -zkvyper -undol -applyl -Upgradability -Initializable -Hola -mundo -ISTN -Zerion -Maverik -zk_evm_1_3_3 -vk -vks -CORS -verifier -crypto -callee -Subcalls -Vec -vec -vecs -L1Messenger -SystemL2ToL1Log -witness_inputs -StateKeeper -enum_index -virtual_block_start_batch -virtual_block_finish_l2_block -base_token_address -maxFeePerGas -maxPriorityFeePerGas -structs -all_circuit -OversizedData -M5 -eth_sign -geth -reth -ethers -js -recovery_id -&self -ETHSignature -recover_signer -BlockNumber -(de) -{result -DebugCall} -CREATE2 -memtables -memtable -PostgreSQL -OneTx -DefaultTracer -Tx1 -Tx2 -TxN -VmStopped -Unversioned -versioned -l2_block -submodule -enums -deserialized -deserialize -hashmap -vm_m5 -SDK -1M -dir -SSD -getter -Getters -WebSocket -gasLimit -MiBs -MiB -GiB -GiBs -pubsub -\x19Ethereum -nibbles–node -ZkSyncTree -invariants -LEB128 -workflow -L1Batch -runtime -Tokio -Blobstore -S3 -AWS -ExternalIO -ClosedFormInputWrapper -AggregationWrapper -(de)serializer -typesafe -LRU -ns -Q3 -loadnext -args -with_arg -node_aggregation_job -scheduler_job -leaf_aggregation_job -MAX_ATTEMPTs -fsync -TEST_DATABASE_URL -newest_block -block_count -contracts_verification_info -RNG -jsonrpsee -l1_batch -Namespace -ExecutionStatus -VmStarted -reproducibility -CFs -key–value -enum_index_migration_cursor -block_number -initial_writes -errored -FactoryDeps -de -StorageView's -Yul -eth_txs -eth_tx -ExecuteBlock -PublishProofBlocksOnchain -CommitBlocks -entrypoint -gas_limit -TxSender -UX -BasicWitnessInputProducer -eth_tx_history -PENDING_BLOCK -from_block -namespace -PriorityQueue -Görli -Ropsten -Rinkeby -tokio -threadpool -IntrinsicGas -InsufficientFundsForTransfer -ChainId -hyperchains -eth_getLogs -façade -virtual_blocks_per_miniblock -virtual_block_interval -max_overhead -total_gas_limit -cloneable -timestamped -healthcheck -Healthcheck -HealthCheck -readonly -upgrader -startup -BFT -PingCAP -witgen -ok -hacky -ceil -Infura -synth -proto - -AUTOGENERATED -x19Ethereum -block_timestamp -SYSTEM_BLOCK_INFO_BLOCK_NUMBER_MULTIPLIER -MAX_L2_TX_GAS_LIMIT -MAX_TX_ERGS_LIMIT -OneTxTracer -multicall -Multicall's -Multicall3 -proxied -scalers -updatable -instantiation -unexecuted -transactional -benchmarking -virtual_blocks_interval -dal -codebase -compactions -M6 -compiler_common -noop -tokenized -rustc -sqlx -zkevm -Boojum -Sepolia -psql -Cuda -cuda -hdcaa -impls -abda -edaf -unsynchronized -CUDA -gcloud -NVME -OTLP -multiVM -Deduplicator -lobkc -sread -myfunction -merklelization -beaf -subcalls -unallowed -Nuxt -Merklized -satisfiability -demultiplex -precompile -statekeeper -matchers -lifecycle -dedup -deduped -crаsh -protobuf -L1Tx -EIP -DecommittmentProcessor -decommitment -tokenized -Aggregator -DecommittmentProcessor -decommitment -hardcoded -plookup -shivini -EIP4844 -KZG -secp256k1 -vendoring -publickey -keypair -Electrum -healthcheck -healthchecks -after_node_shutdown -runnable -downcasting -parameterized -reimplementation -composability -md5 -shivini -balancer -lookups -stateful -getPubdataPricingMode -Uint -implementors -WIP -oneshot -p2p -StorageProcessor -StorageMarker -SIGINT -opentelemetry -PubdataSendingMode -FriGpuProverArchiver -vm -demuxer -2k -4k -superset -80M -780kb -None -Nones -evm_simulator_code_hash -pubdata_costs -storage_refunds -state_keeper's -witness_generator -arity -recursion_tip -RECURSION_TIP_ARITY -empty_proof -hyperchain -storages -vec -zksync_merkle_tree -TreeMetadata -delegator -decrement -whitelisted -Bbellman -Sbellman -DCMAKE -preloaded -e2e -upcasting -foundryup -uncached -untrimmed -UNNEST -semver -TeeRequestProcessor -l1_batch_number -RequestProcessorError -map_err -proof_inputs -submit_proofs -ready_to_be_proven -privkey diff --git a/checks-config/links.json b/checks-config/links.json deleted file mode 100644 index b18b9608f16f..000000000000 --- a/checks-config/links.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "ignorePatterns": [ - { - "pattern": "^https://github\\.com/matter-labs/zksync-2-dev/" - }, - { - "pattern": "^https://www\\.notion\\.so/" - }, - { - "pattern": "^https://github\\.com/matter-labs/zksync-era/compare/" - }, - { - "pattern": "^https://twitter\\.com/zksync" - }, - { - "pattern": "^https://twitter\\.com/zkSyncDevs" - }, - { - "pattern": "^https://github\\.com/matter-labs/zk_evm" - }, - { - "pattern": "^https://sepolia\\.etherscan\\.io/tx/0x18c2a113d18c53237a4056403047ff9fafbf772cb83ccd44bb5b607f8108a64c" - }, - { - "pattern": "^https://github\\.com/matter-labs/zksync-era/commit/" - }, - { - "pattern": "^https://github\\.com/matter-labs//era-compiler-llvm" - } - ], - "aliveStatusCodes": [0, 200, 206, 304] -} diff --git a/docker/zk-environment/Dockerfile b/docker/zk-environment/Dockerfile index 9c9393ed5188..c5cb35cf1a07 100644 --- a/docker/zk-environment/Dockerfile +++ b/docker/zk-environment/Dockerfile @@ -84,7 +84,7 @@ RUN mkdir -p /etc/apt/keyrings && \ wget -c -O - https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && \ echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && \ apt-get update && apt-get install nodejs -y && \ - npm install -g yarn && npm install -g cspell + npm install -g yarn # Install Rust and required cargo packages ENV RUSTUP_HOME=/usr/local/rustup \ @@ -105,7 +105,6 @@ RUN wget -c -O - https://sh.rustup.rs | bash -s -- -y && \ RUN cargo install --version=0.7.3 sqlx-cli RUN cargo install cargo-nextest -RUN cargo install cargo-spellcheck # Copy compiler (both solc and zksolc) binaries # Obtain `solc` 0.8.20. diff --git a/docs/guides/development.md b/docs/guides/development.md index 5e53877993dd..c859017848b5 100644 --- a/docs/guides/development.md +++ b/docs/guides/development.md @@ -89,56 +89,6 @@ Currently the following criteria are checked: - Other code should always be formatted via `zk fmt`. - Dummy Prover should not be staged for commit (see below for the explanation). -## Spell Checking - -In our development workflow, we utilize a spell checking process to ensure the quality and accuracy of our documentation -and code comments. This is achieved using two primary tools: `cspell` and `cargo-spellcheck`. This section outlines how -to use these tools and configure them for your needs. - -### Using the Spellcheck Command - -The spell check command `zk spellcheck` is designed to check for spelling errors in our documentation and code. To run -the spell check, use the following command: - -``` -zk spellcheck -Options: ---pattern : Specifies the glob pattern for files to check. Default is docs/**/*. ---use-cargo: Utilize cargo spellcheck. ---use-cspell: Utilize cspell. -``` - -### General Rules - -**Code References in Comments**: When referring to code elements within development comments, they should be wrapped in -backticks. For example, reference a variable as `block_number`. - -**Code Blocks in Comments**: For larger blocks of pseudocode or commented-out code, use code blocks formatted as -follows: - -```` -// ``` -// let overhead_for_pubdata = { -// let numerator: U256 = overhead_for_block_gas * total_gas_limit -// + gas_per_pubdata_byte_limit * U256::from(MAX_PUBDATA_PER_BLOCK); -// let denominator = -// gas_per_pubdata_byte_limit * U256::from(MAX_PUBDATA_PER_BLOCK) + overhead_for_block_gas; -// ``` -```` - -**Language Settings**: We use the Hunspell language setting of `en_US`. - -**CSpell Usage**: For spell checking within the `docs/` directory, we use `cspell`. The configuration for this tool is -found in `cspell.json`. It's tailored to check our documentation for spelling errors. - -**Cargo-Spellcheck for Rust and Dev Comments**: For Rust code and development comments, `cargo-spellcheck` is used. Its -configuration is maintained in `era.cfg`. - -### Adding Words to the Dictionary - -To add a new word to the spell checker dictionary, navigate to `/spellcheck/era.dic` and include the word. Ensure that -the word is relevant and necessary to be included in the dictionary to maintain the integrity of our documentation. - ## Using Dummy Prover By default, the chosen prover is a "dummy" one, meaning that it doesn't actually compute proofs but rather uses mocks to diff --git a/infrastructure/zk/package.json b/infrastructure/zk/package.json index dc6aded093a2..29d47184fa09 100644 --- a/infrastructure/zk/package.json +++ b/infrastructure/zk/package.json @@ -31,7 +31,6 @@ "@types/tabtab": "^3.0.1", "hardhat": "=2.22.2", "typescript": "^4.3.5", - "cspell": "^8.3.2", "sql-formatter": "^13.1.0" } } diff --git a/infrastructure/zk/src/index.ts b/infrastructure/zk/src/index.ts index 0c11c110c6e3..5aef41cca388 100644 --- a/infrastructure/zk/src/index.ts +++ b/infrastructure/zk/src/index.ts @@ -23,7 +23,6 @@ import { command as db } from './database'; import { command as verifyUpgrade } from './verify-upgrade'; import { proverCommand } from './prover_setup'; import { command as status } from './status'; -import { command as spellcheck } from './spellcheck'; import { command as setupEn } from './setup_en'; import * as env from './env'; @@ -50,7 +49,6 @@ const COMMANDS = [ proverCommand, env.command, status, - spellcheck, setupEn, completion(program as Command) ]; diff --git a/infrastructure/zk/src/spellcheck.ts b/infrastructure/zk/src/spellcheck.ts deleted file mode 100644 index 8bf78869788d..000000000000 --- a/infrastructure/zk/src/spellcheck.ts +++ /dev/null @@ -1,44 +0,0 @@ -import { Command } from 'commander'; -import * as utils from 'utils'; - -export async function runSpellCheck(pattern: string, useCargo: boolean, useCSpell: boolean) { - // Default commands for cSpell and cargo spellcheck - const cSpellCommand = `cspell "${pattern}" --config=./checks-config/cspell.json`; - const cargoCommand = `cargo spellcheck --cfg=./checks-config/era.cfg --code 1`; - // Necessary to run cargo spellcheck in the prover directory explicitly as - // it is not included in the root cargo.toml file - const cargoCommandForProver = `cargo spellcheck --cfg=../checks-config/era.cfg --code 1`; - - try { - let results = []; - - // Run cspell over all **/*.md files - if (useCSpell || (!useCargo && !useCSpell)) { - results.push(await utils.spawn(cSpellCommand)); - } - - // Run cargo spellcheck in core and prover directories - if (useCargo || (!useCargo && !useCSpell)) { - results.push(await utils.spawn(cargoCommand)); - results.push(await utils.spawn('cd prover && ' + cargoCommandForProver)); - } - - // Check results and exit with error code if any command failed - if (results.some((code) => code !== 0)) { - console.error('Spell check failed'); - process.exit(1); - } - } catch (error) { - console.error('Error occurred during spell checking:', error); - process.exit(1); - } -} - -export const command = new Command('spellcheck') - .option('--pattern ', 'Glob pattern for files to check', '**/*.md') - .option('--use-cargo', 'Use cargo spellcheck') - .option('--use-cspell', 'Use cspell') - .description('Run spell check on specified files') - .action((cmd) => { - runSpellCheck(cmd.pattern, cmd.useCargo, cmd.useCSpell); - }); From 06c287b630707843fd92cb88f899a8fd1dcc7147 Mon Sep 17 00:00:00 2001 From: AnastasiiaVashchuk <72273339+AnastasiiaVashchuk@users.noreply.github.com> Date: Fri, 21 Jun 2024 08:38:51 +0300 Subject: [PATCH 38/43] feat(docs): Add documentation for subset of wiring layer implementations, used by Main node (#2292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is the first PR in the queue. ## What ❔ Adds a description for the next wiring layers: - `CircuitBreakerCheckerLayer` - `CommitmentGeneratorLayer` - `ContractVerificationApiLayer` - `EthTxManagerLayer` - `EthTxAggregatorLayer` - `EthWatchLayer` - `HealthCheckLayer` --- .../layers/circuit_breaker_checker.rs | 11 +++++++ .../layers/commitment_generator.rs | 10 ++++++ .../layers/contract_verification_api.rs | 10 ++++++ .../src/implementations/layers/eth_sender.rs | 31 +++++++++++++++++++ .../src/implementations/layers/eth_watch.rs | 11 +++++++ .../layers/healtcheck_server.rs | 17 +++++----- 6 files changed, 81 insertions(+), 9 deletions(-) diff --git a/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs b/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs index b8fff34b7e92..52e72519110b 100644 --- a/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs +++ b/core/node/node_framework/src/implementations/layers/circuit_breaker_checker.rs @@ -8,6 +8,17 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; +/// Wiring layer for circuit breaker checker +/// +/// Expects other layers to insert different components' circuit breakers into +/// [`zksync_circuit_breaker::CircuitBreakers`] collection using [`CircuitBreakersResource`]. +/// The added task periodically runs checks for all inserted circuit breakers. +/// +/// ## Adds resources +/// - [`CircuitBreakersResource`] +/// +/// ## Adds tasks +/// - [`CircuitBreakerCheckerTask`] (as [`UnconstrainedTask`]) #[derive(Debug)] pub struct CircuitBreakerCheckerLayer(pub CircuitBreakerConfig); diff --git a/core/node/node_framework/src/implementations/layers/commitment_generator.rs b/core/node/node_framework/src/implementations/layers/commitment_generator.rs index cc57599759eb..ccbafba1d717 100644 --- a/core/node/node_framework/src/implementations/layers/commitment_generator.rs +++ b/core/node/node_framework/src/implementations/layers/commitment_generator.rs @@ -13,6 +13,16 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; +/// Wiring layer for l1 batches commitment generation +/// +/// Responsible for initialization and running [`CommitmentGenerator`]. +/// +/// ## Requests resources +/// - [`PoolResource`] for [`MasterPool`] +/// - [`AppHealthCheckResource`] (to add new health check) +/// +/// ## Adds tasks +/// - [`CommitmentGeneratorTask`] (as [`Task`]) #[derive(Debug)] pub struct CommitmentGeneratorLayer { mode: L1BatchCommitmentMode, diff --git a/core/node/node_framework/src/implementations/layers/contract_verification_api.rs b/core/node/node_framework/src/implementations/layers/contract_verification_api.rs index 5e76c32ddd53..3d26333c00a8 100644 --- a/core/node/node_framework/src/implementations/layers/contract_verification_api.rs +++ b/core/node/node_framework/src/implementations/layers/contract_verification_api.rs @@ -8,6 +8,16 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; +/// Wiring layer for contract verification +/// +/// Responsible for initialization of the contract verification server. +/// +/// ## Requests resources +/// - [`PoolResource`] for [`MasterPool`] +/// - [`PoolResource`] for [`ReplicaPool`] +/// +/// ## Adds tasks +/// - [`ContractVerificationApiTask`] (as [`Task`]) #[derive(Debug)] pub struct ContractVerificationApiLayer(pub ContractVerifierConfig); diff --git a/core/node/node_framework/src/implementations/layers/eth_sender.rs b/core/node/node_framework/src/implementations/layers/eth_sender.rs index 3cf2cf597c31..677d86560737 100644 --- a/core/node/node_framework/src/implementations/layers/eth_sender.rs +++ b/core/node/node_framework/src/implementations/layers/eth_sender.rs @@ -18,6 +18,21 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; +/// Wiring layer for `eth_txs` managing +/// +/// Responsible for initialization and running [`EthTxManager`] component, that manages sending +/// of `eth_txs`(such as `CommitBlocks`, `PublishProofBlocksOnchain` or `ExecuteBlock` ) to L1. +/// +/// ## Requests resources +/// - [`PoolResource`] for [`MasterPool`] +/// - [`PoolResource`] for [`ReplicaPool`] +/// - [`BoundEthInterfaceResource`] +/// - [`BoundEthInterfaceForBlobsResource`] +/// - [`L1TxParamsResource`] +/// - [`CircuitBreakersResource`] (to add new circuit breaker) +/// +/// ## Adds tasks +/// - [`EthTxManagerTask`] (as [`Task`]) #[derive(Debug)] pub struct EthTxManagerLayer { eth_sender_config: EthConfig, @@ -78,6 +93,22 @@ impl WiringLayer for EthTxManagerLayer { } } +/// Wiring layer for aggregating l1 batches into `eth_txs` +/// +/// Responsible for initialization and running of [`EthTxAggregator`], that aggregates L1 batches +/// into `eth_txs`(such as `CommitBlocks`, `PublishProofBlocksOnchain` or `ExecuteBlock`). +/// These `eth_txs` will be used as a queue for generating signed txs and will be sent later on L1. +/// +/// ## Requests resources +/// - [`PoolResource`] for [`MasterPool`] +/// - [`PoolResource`] for [`ReplicaPool`] +/// - [`BoundEthInterfaceResource`] +/// - [`BoundEthInterfaceForBlobsResource`] +/// - [`ObjectStoreResource`] +/// - [`CircuitBreakersResource`] (to add new circuit breaker) +/// +/// ## Adds tasks +/// - [`EthTxAggregatorTask`] (as [`Task`]) #[derive(Debug)] pub struct EthTxAggregatorLayer { eth_sender_config: EthConfig, diff --git a/core/node/node_framework/src/implementations/layers/eth_watch.rs b/core/node/node_framework/src/implementations/layers/eth_watch.rs index df9319013112..809da037d97a 100644 --- a/core/node/node_framework/src/implementations/layers/eth_watch.rs +++ b/core/node/node_framework/src/implementations/layers/eth_watch.rs @@ -16,6 +16,17 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; +/// Wiring layer for ethereum watcher +/// +/// Responsible for initializing and running of [`EthWatch`] component, that polls the Ethereum node for the relevant events, +/// such as priority operations (aka L1 transactions), protocol upgrades etc. +/// +/// ## Requests resources +/// - [`PoolResource`] for [`MasterPool`] +/// - [`EthInterfaceResource`] +/// +/// ## Adds tasks +/// - [`EthWatchTask`] (as [`Task`]) #[derive(Debug)] pub struct EthWatchLayer { eth_watch_config: EthWatchConfig, diff --git a/core/node/node_framework/src/implementations/layers/healtcheck_server.rs b/core/node/node_framework/src/implementations/layers/healtcheck_server.rs index c6138c711083..1ae2b1f54736 100644 --- a/core/node/node_framework/src/implementations/layers/healtcheck_server.rs +++ b/core/node/node_framework/src/implementations/layers/healtcheck_server.rs @@ -11,17 +11,17 @@ use crate::{ wiring_layer::{WiringError, WiringLayer}, }; -/// Builder for a health check server. +/// Wiring layer for health check server /// -/// Spawned task collects all the health checks added by different tasks to the -/// corresponding resource collection and spawns an HTTP server exposing them. +/// Expects other layers to insert different components' health checks +/// into [`AppHealthCheck`] aggregating heath using [`AppHealthCheckResource`]. +/// The added task spawns a health check server that only exposes the state provided by other tasks. /// -/// This layer expects other tasks to add health checks to the `ResourceCollection`. +/// ## Adds resources +/// - [`AppHealthCheckResource`] /// -/// ## Effects -/// -/// - Resolves `ResourceCollection`. -/// - Adds `healthcheck_server` to the node. +/// ## Adds tasks +/// - [`HealthCheckTask`] (as [`UnconstrainedTask`]) #[derive(Debug)] pub struct HealthCheckLayer(pub HealthCheckConfig); @@ -39,7 +39,6 @@ impl WiringLayer for HealthCheckLayer { app_health_check, }; - // Healthcheck server only exposes the state provided by other tasks, and also it has to start as soon as possible. node.add_unconstrained_task(Box::new(task)); Ok(()) } From c2412cf2421448c706a08e3c8fda3b0af6aac497 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 21 Jun 2024 09:02:16 +0300 Subject: [PATCH 39/43] fix(db): Fix `insert_proof_generation_details()` (#2291) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Removes a double insertion check from `insert_proof_generation_details()` in `ProofGenerationDal`. ## Why ❔ It is not an error, and can and will happen if multiple full trees are run for the same node. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Tests for the changes have been added / updated. - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. - [x] Spellcheck has been run via `zk spellcheck`. --- core/lib/dal/src/proof_generation_dal.rs | 133 ++++++++++++++++--- core/lib/db_connection/src/instrument.rs | 39 ++++-- core/node/metadata_calculator/src/updater.rs | 3 +- 3 files changed, 145 insertions(+), 30 deletions(-) diff --git a/core/lib/dal/src/proof_generation_dal.rs b/core/lib/dal/src/proof_generation_dal.rs index 040b4246604f..88300cf08a18 100644 --- a/core/lib/dal/src/proof_generation_dal.rs +++ b/core/lib/dal/src/proof_generation_dal.rs @@ -3,7 +3,9 @@ use std::time::Duration; use strum::{Display, EnumString}; use zksync_db_connection::{ - connection::Connection, error::DalResult, instrument::Instrumented, + connection::Connection, + error::DalResult, + instrument::{InstrumentExt, Instrumented}, utils::pg_interval_from_duration, }; use zksync_types::L1BatchNumber; @@ -110,13 +112,13 @@ impl ProofGenerationDal<'_, '_> { Ok(()) } + /// The caller should ensure that `l1_batch_number` exists in the database. pub async fn insert_proof_generation_details( &mut self, - block_number: L1BatchNumber, + l1_batch_number: L1BatchNumber, proof_gen_data_blob_url: &str, ) -> DalResult<()> { - let l1_batch_number = i64::from(block_number.0); - let query = sqlx::query!( + let result = sqlx::query!( r#" INSERT INTO proof_generation_details (l1_batch_number, status, proof_gen_data_blob_url, created_at, updated_at) @@ -124,25 +126,22 @@ impl ProofGenerationDal<'_, '_> { ($1, 'ready_to_be_proven', $2, NOW(), NOW()) ON CONFLICT (l1_batch_number) DO NOTHING "#, - l1_batch_number, + i64::from(l1_batch_number.0), proof_gen_data_blob_url, - ); - let instrumentation = Instrumented::new("insert_proof_generation_details") - .with_arg("l1_batch_number", &l1_batch_number) - .with_arg("proof_gen_data_blob_url", &proof_gen_data_blob_url); - let result = instrumentation - .clone() - .with(query) - .execute(self.storage) - .await?; + ) + .instrument("insert_proof_generation_details") + .with_arg("l1_batch_number", &l1_batch_number) + .with_arg("proof_gen_data_blob_url", &proof_gen_data_blob_url) + .report_latency() + .execute(self.storage) + .await?; + if result.rows_affected() == 0 { - let err = instrumentation.constraint_error(anyhow::anyhow!( - "Cannot save proof_blob_url for a batch number {} that does not exist", - l1_batch_number - )); - return Err(err); + // Not an error: we may call `insert_proof_generation_details()` from multiple full trees instantiated + // for the same node. Unlike tree data, we don't particularly care about correspondence of `proof_gen_data_blob_url` across calls, + // so just log this fact and carry on. + tracing::debug!("L1 batch #{l1_batch_number}: proof generation data wasn't updated as it's already present"); } - Ok(()) } @@ -229,3 +228,97 @@ impl ProofGenerationDal<'_, '_> { Ok(result) } } + +#[cfg(test)] +mod tests { + use zksync_types::ProtocolVersion; + + use super::*; + use crate::{tests::create_l1_batch_header, ConnectionPool, CoreDal}; + + #[tokio::test] + async fn proof_generation_workflow() { + let pool = ConnectionPool::::test_pool().await; + let mut conn = pool.connection().await.unwrap(); + + conn.protocol_versions_dal() + .save_protocol_version_with_tx(&ProtocolVersion::default()) + .await + .unwrap(); + conn.blocks_dal() + .insert_mock_l1_batch(&create_l1_batch_header(1)) + .await + .unwrap(); + + let unpicked_l1_batch = conn + .proof_generation_dal() + .get_oldest_unpicked_batch() + .await + .unwrap(); + assert_eq!(unpicked_l1_batch, None); + + conn.proof_generation_dal() + .insert_proof_generation_details(L1BatchNumber(1), "generation_data") + .await + .unwrap(); + + let unpicked_l1_batch = conn + .proof_generation_dal() + .get_oldest_unpicked_batch() + .await + .unwrap(); + assert_eq!(unpicked_l1_batch, Some(L1BatchNumber(1))); + + // Calling the method multiple times should work fine. + conn.proof_generation_dal() + .insert_proof_generation_details(L1BatchNumber(1), "generation_data") + .await + .unwrap(); + + let unpicked_l1_batch = conn + .proof_generation_dal() + .get_oldest_unpicked_batch() + .await + .unwrap(); + assert_eq!(unpicked_l1_batch, Some(L1BatchNumber(1))); + + let picked_l1_batch = conn + .proof_generation_dal() + .get_next_block_to_be_proven(Duration::MAX) + .await + .unwrap(); + assert_eq!(picked_l1_batch, Some(L1BatchNumber(1))); + let unpicked_l1_batch = conn + .proof_generation_dal() + .get_oldest_unpicked_batch() + .await + .unwrap(); + assert_eq!(unpicked_l1_batch, None); + + // Check that with small enough processing timeout, the L1 batch can be picked again + let picked_l1_batch = conn + .proof_generation_dal() + .get_next_block_to_be_proven(Duration::ZERO) + .await + .unwrap(); + assert_eq!(picked_l1_batch, Some(L1BatchNumber(1))); + + conn.proof_generation_dal() + .save_proof_artifacts_metadata(L1BatchNumber(1), "proof") + .await + .unwrap(); + + let picked_l1_batch = conn + .proof_generation_dal() + .get_next_block_to_be_proven(Duration::MAX) + .await + .unwrap(); + assert_eq!(picked_l1_batch, None); + let unpicked_l1_batch = conn + .proof_generation_dal() + .get_oldest_unpicked_batch() + .await + .unwrap(); + assert_eq!(unpicked_l1_batch, None); + } +} diff --git a/core/lib/db_connection/src/instrument.rs b/core/lib/db_connection/src/instrument.rs index e0728ce22b85..91f207838c3b 100644 --- a/core/lib/db_connection/src/instrument.rs +++ b/core/lib/db_connection/src/instrument.rs @@ -200,6 +200,21 @@ impl<'a> InstrumentedData<'a> { } } + fn observe_error(&self, err: &dyn fmt::Display) { + let InstrumentedData { + name, + location, + args, + .. + } = self; + tracing::warn!( + "Query {name}{args} called at {file}:{line} has resulted in error: {err}", + file = location.file(), + line = location.line() + ); + REQUEST_METRICS.request_error[name].inc(); + } + async fn fetch( self, connection_tags: Option<&ConnectionTags>, @@ -295,32 +310,40 @@ impl<'a> Instrumented<'a, ()> { } } - /// Wraps a provided argument validation error. + /// Wraps a provided argument validation error. It is assumed that the returned error + /// will be returned as an error cause (e.g., it is logged as an error and observed using metrics). + #[must_use] pub fn arg_error(&self, arg_name: &str, err: E) -> DalError where E: Into, { let err: anyhow::Error = err.into(); let err = err.context(format!("failed validating query argument `{arg_name}`")); - DalRequestError::new( + let err = DalRequestError::new( sqlx::Error::Decode(err.into()), self.data.name, self.data.location, ) - .with_args(self.data.args.to_owned()) - .into() + .with_args(self.data.args.to_owned()); + + self.data.observe_error(&err); + err.into() } - /// Wraps a provided application-level data constraint error. + /// Wraps a provided application-level data constraint error. It is assumed that the returned error + /// will be returned as an error cause (e.g., it is logged as an error and observed using metrics). + #[must_use] pub fn constraint_error(&self, err: anyhow::Error) -> DalError { let err = err.context("application-level data constraint violation"); - DalRequestError::new( + let err = DalRequestError::new( sqlx::Error::Decode(err.into()), self.data.name, self.data.location, ) - .with_args(self.data.args.to_owned()) - .into() + .with_args(self.data.args.to_owned()); + + self.data.observe_error(&err); + err.into() } pub fn with(self, query: Q) -> Instrumented<'a, Q> { diff --git a/core/node/metadata_calculator/src/updater.rs b/core/node/metadata_calculator/src/updater.rs index 2056b8315664..bfb6ad1912a0 100644 --- a/core/node/metadata_calculator/src/updater.rs +++ b/core/node/metadata_calculator/src/updater.rs @@ -145,8 +145,7 @@ impl TreeUpdater { storage .tee_verifier_input_producer_dal() .create_tee_verifier_input_producer_job(l1_batch_number) - .await - .expect("failed to create tee_verifier_input_producer job"); + .await?; // Save the proof generation details to Postgres storage .proof_generation_dal() From e467028d2d7a27f9a076cc559c5a6fa1c7605b5a Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 21 Jun 2024 13:43:26 +0300 Subject: [PATCH 40/43] refactor(pruning): Improve pruning metrics and logs (#2297) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ - Adds counters for outcomes (success, failure, error) for all pruning conditions. - Distinguishes between no-op and soft pruning latency. - Logs latencies and stats in tree pruning. - Other misc improvements. ## Why ❔ Improves pruning observability. ## Checklist - [x] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [x] Documentation comments have been added / updated. - [x] Code has been formatted via `zk fmt` and `zk lint`. --- core/lib/merkle_tree/src/metrics.rs | 34 +++++++++---------- core/lib/merkle_tree/src/pruning.rs | 8 +++-- core/node/db_pruner/src/lib.rs | 33 +++++++++++++------ core/node/db_pruner/src/metrics.rs | 36 +++++++++++++++++++-- core/node/db_pruner/src/prune_conditions.rs | 22 +++++++++++++ core/node/db_pruner/src/tests.rs | 6 +++- 6 files changed, 105 insertions(+), 34 deletions(-) diff --git a/core/lib/merkle_tree/src/metrics.rs b/core/lib/merkle_tree/src/metrics.rs index 84769482527a..99757a2580cb 100644 --- a/core/lib/merkle_tree/src/metrics.rs +++ b/core/lib/merkle_tree/src/metrics.rs @@ -309,6 +309,21 @@ enum Bound { End, } +const LARGE_NODE_COUNT_BUCKETS: Buckets = Buckets::values(&[ + 1_000.0, + 2_000.0, + 5_000.0, + 10_000.0, + 20_000.0, + 50_000.0, + 100_000.0, + 200_000.0, + 500_000.0, + 1_000_000.0, + 2_000_000.0, + 5_000_000.0, +]); + #[derive(Debug, Metrics)] #[metrics(prefix = "merkle_tree_pruning")] struct PruningMetrics { @@ -316,7 +331,7 @@ struct PruningMetrics { /// may not remove all stale keys to this version if there are too many. target_retained_version: Gauge, /// Number of pruned node keys on a specific pruning iteration. - #[metrics(buckets = NODE_COUNT_BUCKETS)] + #[metrics(buckets = LARGE_NODE_COUNT_BUCKETS)] key_count: Histogram, /// Lower and upper boundaries on the new stale key versions deleted /// during a pruning iteration. The lower boundary is inclusive, the upper one is exclusive. @@ -368,26 +383,11 @@ pub(crate) enum RecoveryStage { ParallelPersistence, } -const CHUNK_SIZE_BUCKETS: Buckets = Buckets::values(&[ - 1_000.0, - 2_000.0, - 5_000.0, - 10_000.0, - 20_000.0, - 50_000.0, - 100_000.0, - 200_000.0, - 500_000.0, - 1_000_000.0, - 2_000_000.0, - 5_000_000.0, -]); - #[derive(Debug, Metrics)] #[metrics(prefix = "merkle_tree_recovery")] pub(crate) struct RecoveryMetrics { /// Number of entries in a recovered chunk. - #[metrics(buckets = CHUNK_SIZE_BUCKETS)] + #[metrics(buckets = LARGE_NODE_COUNT_BUCKETS)] pub chunk_size: Histogram, /// Latency of a specific stage of recovery for a single chunk. #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)] diff --git a/core/lib/merkle_tree/src/pruning.rs b/core/lib/merkle_tree/src/pruning.rs index 1734fdcbf0a8..a74db40ef5e6 100644 --- a/core/lib/merkle_tree/src/pruning.rs +++ b/core/lib/merkle_tree/src/pruning.rs @@ -166,7 +166,7 @@ impl MerkleTreePruner { break; } } - load_stale_keys_latency.observe(); + let load_stale_keys_latency = load_stale_keys_latency.observe(); if pruned_keys.is_empty() { tracing::debug!("No stale keys to remove; skipping"); @@ -174,7 +174,7 @@ impl MerkleTreePruner { } let deleted_stale_key_versions = min_stale_key_version..(max_stale_key_version + 1); tracing::info!( - "Collected {} stale keys with new versions in {deleted_stale_key_versions:?}", + "Collected {} stale keys with new versions in {deleted_stale_key_versions:?} in {load_stale_keys_latency:?}", pruned_keys.len() ); @@ -186,7 +186,8 @@ impl MerkleTreePruner { let patch = PrunePatchSet::new(pruned_keys, deleted_stale_key_versions); let apply_patch_latency = PRUNING_TIMINGS.apply_patch.start(); self.db.prune(patch)?; - apply_patch_latency.observe(); + let apply_patch_latency = apply_patch_latency.observe(); + tracing::info!("Pruned stale keys in {apply_patch_latency:?}: {stats:?}"); Ok(Some(stats)) } @@ -230,6 +231,7 @@ impl MerkleTreePruner { self.poll_interval }; } + tracing::info!("Stop signal received, tree pruning is shut down"); Ok(()) } } diff --git a/core/node/db_pruner/src/lib.rs b/core/node/db_pruner/src/lib.rs index 22a1e4453614..4b4a53c68aa0 100644 --- a/core/node/db_pruner/src/lib.rs +++ b/core/node/db_pruner/src/lib.rs @@ -1,6 +1,9 @@ //! Postgres pruning component. -use std::{sync::Arc, time::Duration}; +use std::{ + sync::Arc, + time::{Duration, Instant}, +}; use anyhow::Context as _; use serde::{Deserialize, Serialize}; @@ -10,7 +13,7 @@ use zksync_health_check::{Health, HealthStatus, HealthUpdater, ReactiveHealthChe use zksync_types::{L1BatchNumber, L2BlockNumber}; use self::{ - metrics::{MetricPruneType, METRICS}, + metrics::{ConditionOutcome, PruneType, METRICS}, prune_conditions::{ ConsistencyCheckerProcessedBatch, L1BatchExistsCondition, L1BatchOlderThanPruneCondition, NextL1BatchHasMetadataCondition, NextL1BatchWasExecutedCondition, PruneCondition, @@ -128,15 +131,24 @@ impl DbPruner { let mut errored_conditions = vec![]; for condition in &self.prune_conditions { - match condition.is_batch_prunable(l1_batch_number).await { - Ok(true) => successful_conditions.push(condition.to_string()), - Ok(false) => failed_conditions.push(condition.to_string()), + let outcome = match condition.is_batch_prunable(l1_batch_number).await { + Ok(true) => { + successful_conditions.push(condition.to_string()); + ConditionOutcome::Success + } + Ok(false) => { + failed_conditions.push(condition.to_string()); + ConditionOutcome::Fail + } Err(error) => { errored_conditions.push(condition.to_string()); tracing::warn!("Pruning condition '{condition}' resulted in an error: {error}"); + ConditionOutcome::Error } - } + }; + METRICS.observe_condition(condition.as_ref(), outcome); } + let result = failed_conditions.is_empty() && errored_conditions.is_empty(); if !result { tracing::debug!( @@ -172,7 +184,7 @@ impl DbPruner { } async fn soft_prune(&self, storage: &mut Connection<'_, Core>) -> anyhow::Result { - let latency = METRICS.pruning_chunk_duration[&MetricPruneType::Soft].start(); + let start = Instant::now(); let mut transaction = storage.start_transaction().await?; let mut current_pruning_info = transaction.pruning_dal().get_pruning_info().await?; @@ -184,7 +196,7 @@ impl DbPruner { + self.config.pruned_batch_chunk_size, ); if !self.is_l1_batch_prunable(next_l1_batch_to_prune).await { - latency.observe(); + METRICS.pruning_chunk_duration[&PruneType::NoOp].observe(start.elapsed()); return Ok(false); } @@ -200,7 +212,8 @@ impl DbPruner { transaction.commit().await?; - let latency = latency.observe(); + let latency = start.elapsed(); + METRICS.pruning_chunk_duration[&PruneType::Soft].observe(latency); tracing::info!( "Soft pruned db l1_batches up to {next_l1_batch_to_prune} and L2 blocks up to {next_l2_block_to_prune}, operation took {latency:?}", ); @@ -216,7 +229,7 @@ impl DbPruner { storage: &mut Connection<'_, Core>, stop_receiver: &mut watch::Receiver, ) -> anyhow::Result { - let latency = METRICS.pruning_chunk_duration[&MetricPruneType::Hard].start(); + let latency = METRICS.pruning_chunk_duration[&PruneType::Hard].start(); let mut transaction = storage.start_transaction().await?; let mut current_pruning_info = transaction.pruning_dal().get_pruning_info().await?; diff --git a/core/node/db_pruner/src/metrics.rs b/core/node/db_pruner/src/metrics.rs index 0d4d88513dbc..2833bc97f9c1 100644 --- a/core/node/db_pruner/src/metrics.rs +++ b/core/node/db_pruner/src/metrics.rs @@ -1,11 +1,16 @@ use std::time::Duration; -use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit}; +use vise::{ + Buckets, Counter, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Histogram, Metrics, Unit, +}; use zksync_dal::pruning_dal::HardPruningStats; +use crate::prune_conditions::PruneCondition; + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue, EncodeLabelSet)] #[metrics(label = "prune_type", rename_all = "snake_case")] -pub(super) enum MetricPruneType { +pub(super) enum PruneType { + NoOp, Soft, Hard, } @@ -21,8 +26,23 @@ enum PrunedEntityType { CallTrace, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelValue)] +#[metrics(rename_all = "snake_case")] +pub(crate) enum ConditionOutcome { + Success, + Fail, + Error, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, EncodeLabelSet)] +struct ConditionOutcomeLabels { + condition: &'static str, + outcome: ConditionOutcome, +} + const ENTITY_COUNT_BUCKETS: Buckets = Buckets::values(&[ 1.0, 2.0, 5.0, 10.0, 20.0, 50.0, 100.0, 200.0, 500.0, 1_000.0, 2_000.0, 5_000.0, 10_000.0, + 20_000.0, 50_000.0, 100_000.0, ]); #[derive(Debug, Metrics)] @@ -30,12 +50,14 @@ const ENTITY_COUNT_BUCKETS: Buckets = Buckets::values(&[ pub(super) struct DbPrunerMetrics { /// Total latency of pruning chunk of L1 batches. #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)] - pub pruning_chunk_duration: Family>, + pub pruning_chunk_duration: Family>, /// Number of not-pruned L1 batches. pub not_pruned_l1_batches_count: Gauge, /// Number of entities deleted during a single hard pruning iteration, grouped by entity type. #[metrics(buckets = ENTITY_COUNT_BUCKETS)] deleted_entities: Family>, + /// Number of times a certain condition has resulted in a specific outcome (succeeded, failed, or errored). + condition_outcomes: Family, } impl DbPrunerMetrics { @@ -61,6 +83,14 @@ impl DbPrunerMetrics { self.deleted_entities[&PrunedEntityType::L2ToL1Log].observe(deleted_l2_to_l1_logs); self.deleted_entities[&PrunedEntityType::CallTrace].observe(deleted_call_traces); } + + pub fn observe_condition(&self, condition: &dyn PruneCondition, outcome: ConditionOutcome) { + let labels = ConditionOutcomeLabels { + condition: condition.metric_label(), + outcome, + }; + self.condition_outcomes[&labels].inc(); + } } #[vise::register] diff --git a/core/node/db_pruner/src/prune_conditions.rs b/core/node/db_pruner/src/prune_conditions.rs index fef6b57f3352..42f225f4a44b 100644 --- a/core/node/db_pruner/src/prune_conditions.rs +++ b/core/node/db_pruner/src/prune_conditions.rs @@ -7,6 +7,8 @@ use zksync_types::L1BatchNumber; #[async_trait] pub(crate) trait PruneCondition: fmt::Debug + fmt::Display + Send + Sync + 'static { + fn metric_label(&self) -> &'static str; + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result; } @@ -24,6 +26,10 @@ impl fmt::Display for L1BatchOlderThanPruneCondition { #[async_trait] impl PruneCondition for L1BatchOlderThanPruneCondition { + fn metric_label(&self) -> &'static str { + "l1_batch_older_than_minimum_age" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { let mut storage = self.pool.connection_tagged("db_pruner").await?; let l1_batch_header = storage @@ -50,6 +56,10 @@ impl fmt::Display for NextL1BatchWasExecutedCondition { #[async_trait] impl PruneCondition for NextL1BatchWasExecutedCondition { + fn metric_label(&self) -> &'static str { + "next_l1_batch_was_executed" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { let mut storage = self.pool.connection_tagged("db_pruner").await?; let next_l1_batch_number = L1BatchNumber(l1_batch_number.0 + 1); @@ -76,6 +86,10 @@ impl fmt::Display for NextL1BatchHasMetadataCondition { #[async_trait] impl PruneCondition for NextL1BatchHasMetadataCondition { + fn metric_label(&self) -> &'static str { + "next_l1_batch_has_metadata" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { let mut storage = self.pool.connection_tagged("db_pruner").await?; let next_l1_batch_number = L1BatchNumber(l1_batch_number.0 + 1); @@ -117,6 +131,10 @@ impl fmt::Display for L1BatchExistsCondition { #[async_trait] impl PruneCondition for L1BatchExistsCondition { + fn metric_label(&self) -> &'static str { + "l1_batch_exists" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { let mut storage = self.pool.connection_tagged("db_pruner").await?; let l1_batch_header = storage @@ -140,6 +158,10 @@ impl fmt::Display for ConsistencyCheckerProcessedBatch { #[async_trait] impl PruneCondition for ConsistencyCheckerProcessedBatch { + fn metric_label(&self) -> &'static str { + "l1_batch_consistency_checked" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { let mut storage = self.pool.connection_tagged("db_pruner").await?; let last_processed_l1_batch = storage diff --git a/core/node/db_pruner/src/tests.rs b/core/node/db_pruner/src/tests.rs index 9a962d518ec3..d4dbe4546035 100644 --- a/core/node/db_pruner/src/tests.rs +++ b/core/node/db_pruner/src/tests.rs @@ -47,10 +47,14 @@ impl fmt::Display for ConditionMock { #[async_trait] impl PruneCondition for ConditionMock { + fn metric_label(&self) -> &'static str { + "mock" + } + async fn is_batch_prunable(&self, l1_batch_number: L1BatchNumber) -> anyhow::Result { self.is_batch_prunable_responses .get(&l1_batch_number) - .cloned() + .copied() .context("error!") } } From 6fd425813c2b4305de9eabd2bf39850906eb1cad Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 21 Jun 2024 16:28:50 +0400 Subject: [PATCH 41/43] chore: bumped curve25519 (#2299) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ https://github.com/matter-labs/zksync-era/pull/2274 got reverted ## Why ❔ Vuln ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- Cargo.lock | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3582fbe51319..b013517e0cc2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1509,16 +1509,15 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.2" +version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" +checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ "cfg-if 1.0.0", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms", "rustc_version", "subtle", "zeroize", @@ -4387,12 +4386,6 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" -[[package]] -name = "platforms" -version = "3.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6ab3f592e6fb464fc9712d8d6e6912de6473954635fd76a589d832cffcbb0" - [[package]] name = "plotters" version = "0.3.5" From 9303142de5e6af3da69fa836a7e537287bdde4b0 Mon Sep 17 00:00:00 2001 From: Igor Aleksanov Date: Fri, 21 Jun 2024 16:31:59 +0400 Subject: [PATCH 42/43] feat: Use info log level for crates named zksync_* by default (#2296) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Makes `zksync=info` default log entry, which later can be overridden/extended. ## Why ❔ Provides a good enough default for any env, simplifies configuration of the system. ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. --- core/bin/block_reverter/src/main.rs | 4 ++- core/lib/vlog/src/lib.rs | 42 ++++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 7 deletions(-) diff --git a/core/bin/block_reverter/src/main.rs b/core/bin/block_reverter/src/main.rs index 8d1198627a83..53ba90d99ffe 100644 --- a/core/bin/block_reverter/src/main.rs +++ b/core/bin/block_reverter/src/main.rs @@ -92,7 +92,9 @@ async fn main() -> anyhow::Result<()> { .parse() .context("Invalid log format")?; - let mut builder = vlog::ObservabilityBuilder::new().with_log_format(log_format); + let mut builder = vlog::ObservabilityBuilder::new() + .with_log_format(log_format) + .disable_default_logs(); // It's a CLI application, so we only need to show logs that were actually requested. if let Some(sentry_url) = observability_config.sentry_url { builder = builder .with_sentry_url(&sentry_url) diff --git a/core/lib/vlog/src/lib.rs b/core/lib/vlog/src/lib.rs index a65a11f3c479..055011f9606f 100644 --- a/core/lib/vlog/src/lib.rs +++ b/core/lib/vlog/src/lib.rs @@ -139,6 +139,7 @@ pub struct OpenTelemetryOptions { /// Currently capable of configuring logging output and sentry integration. #[derive(Debug, Default)] pub struct ObservabilityBuilder { + disable_default_logs: bool, log_format: LogFormat, log_directives: Option, sentry_url: Option, @@ -176,6 +177,14 @@ impl ObservabilityBuilder { self } + /// Disables logs enabled by default. + /// May be used, for example, in interactive CLI applications, where the user may want to fully control + /// the verbosity. + pub fn disable_default_logs(mut self) -> Self { + self.disable_default_logs = true; + self + } + /// Enables Sentry integration. /// Returns an error if the provided Sentry URL is invalid. pub fn with_sentry_url( @@ -254,15 +263,36 @@ impl ObservabilityBuilder { subscriber.with(layer) } + /// Builds a filter for the logs. + /// + /// Unless `disable_default_logs` was set, uses `zksync=info` as a default which is then merged + /// with user-defined directives. Provided directives can extend/override the default value. + /// + /// The provided default convers all the crates with a name starting with `zksync` (per `tracing` + /// [documentation][1]), which is a good enough default for any project. + /// + /// If `log_directives` are provided via `with_log_directives`, they will be used. + /// Otherwise, the value will be parsed from the environment variable `RUST_LOG`. + /// + /// [1]: https://docs.rs/tracing-subscriber/0.3.18/tracing_subscriber/filter/targets/struct.Targets.html#filtering-with-targets + fn build_filter(&self) -> EnvFilter { + let mut directives = if self.disable_default_logs { + "".to_string() + } else { + "zksync=info,".to_string() + }; + if let Some(log_directives) = &self.log_directives { + directives.push_str(log_directives); + } else if let Ok(env_directives) = std::env::var(EnvFilter::DEFAULT_ENV) { + directives.push_str(&env_directives); + }; + EnvFilter::new(directives) + } + /// Initializes the observability subsystem. pub fn build(self) -> ObservabilityGuard { // Initialize logs. - - let env_filter = if let Some(log_directives) = self.log_directives { - tracing_subscriber::EnvFilter::new(log_directives) - } else { - tracing_subscriber::EnvFilter::from_default_env() - }; + let env_filter = self.build_filter(); match self.log_format { LogFormat::Plain => { From 682a214a193dde0f4f79fd5fa2505406854ad69d Mon Sep 17 00:00:00 2001 From: Danil Date: Fri, 21 Jun 2024 19:35:31 +0200 Subject: [PATCH 43/43] feat(init): Update init scripts (#2301) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What ❔ Update contracts ## Why ❔ ## Checklist - [ ] PR title corresponds to the body of PR (we generate changelog entries from PRs). - [ ] Tests for the changes have been added / updated. - [ ] Documentation comments have been added / updated. - [ ] Code has been formatted via `zk fmt` and `zk lint`. Signed-off-by: Danil --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index 8a70bbbc4812..db9387690502 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit 8a70bbbc48125f5bde6189b4e3c6a3ee79631678 +Subproject commit db9387690502937de081a959b164db5a5262ce0a