From cff17572fee382fc82ec9c868be6a4e3c51a4baa Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:47:29 -0300 Subject: [PATCH 01/26] feat: add state override for gas estimates --- core/lib/types/src/api/mod.rs | 16 ++ core/lib/web3_decl/src/namespaces/eth.rs | 9 +- .../src/api_server/tx_sender/mod.rs | 149 +++++++++++++++++- .../web3/backend_jsonrpsee/namespaces/eth.rs | 13 +- .../src/api_server/web3/namespaces/eth.rs | 12 +- .../src/api_server/web3/namespaces/zks.rs | 2 +- .../src/api_server/web3/tests/vm.rs | 117 +++++++++++++- 7 files changed, 298 insertions(+), 20 deletions(-) diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 9f00aee0cf71..157a00a2cac5 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -1,3 +1,5 @@ +use std::collections::HashMap; + use chrono::{DateTime, Utc}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use strum::Display; @@ -702,3 +704,17 @@ pub struct Proof { pub address: Address, pub storage_proof: Vec, } + +/// Collection of overridden accounts, useful for `eth_estimateGas`. +pub type StateOverride = HashMap; + +/// Account override for `eth_estimateGas`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OverrideAccount { + pub balance: Option, + pub nonce: Option, + pub code: Option, + pub state: Option>, + pub state_diff: Option>, +} diff --git a/core/lib/web3_decl/src/namespaces/eth.rs b/core/lib/web3_decl/src/namespaces/eth.rs index 8fa3b1532056..60c86748d5a4 100644 --- a/core/lib/web3_decl/src/namespaces/eth.rs +++ b/core/lib/web3_decl/src/namespaces/eth.rs @@ -3,7 +3,7 @@ use jsonrpsee::{ proc_macros::rpc, }; use zksync_types::{ - api::{BlockId, BlockIdVariant, BlockNumber, Transaction, TransactionVariant}, + api::{BlockId, BlockIdVariant, BlockNumber, StateOverride, Transaction, TransactionVariant}, transaction_request::CallRequest, Address, H256, }; @@ -36,7 +36,12 @@ pub trait EthNamespace { async fn call(&self, req: CallRequest, block: Option) -> RpcResult; #[method(name = "estimateGas")] - async fn estimate_gas(&self, req: CallRequest, _block: Option) -> RpcResult; + async fn estimate_gas( + &self, + req: CallRequest, + _block: Option, + state_override: Option, + ) -> RpcResult; #[method(name = "gasPrice")] async fn gas_price(&self) -> RpcResult; diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index e0944c42b56b..2ca1f2b315be 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -1,8 +1,9 @@ //! Helper module to submit transactions into the zkSync Network. -use std::{cmp, sync::Arc, time::Instant}; +use std::{cmp, collections::HashMap, sync::Arc, time::Instant}; use anyhow::Context as _; +use itertools::Itertools; use multivm::{ interface::VmExecutionResultAndLogs, utils::{adjust_pubdata_price_for_tx, derive_base_fee_and_gas_per_pubdata, derive_overhead}, @@ -14,17 +15,18 @@ use zksync_dal::{transactions_dal::L2TxSubmissionResult, ConnectionPool, Storage use zksync_state::PostgresStorageCaches; use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ + api::StateOverride, fee::{Fee, TransactionExecutionMetrics}, fee_model::BatchFeeInput, - get_code_key, get_intrinsic_constants, + get_code_key, get_intrinsic_constants, get_nonce_key, l1::is_l1_tx_type, l2::{error::TxCheckError::TxDuplication, L2Tx}, - utils::storage_key_for_eth_balance, + utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, AccountTreeId, Address, ExecuteTransactionCommon, L2ChainId, MiniblockNumber, Nonce, - PackedEthSignature, ProtocolVersionId, Transaction, VmVersion, H160, H256, MAX_L2_TX_GAS_LIMIT, - MAX_NEW_FACTORY_DEPS, U256, + PackedEthSignature, ProtocolVersionId, StorageKey, StorageLog, Transaction, VmVersion, H160, + H256, MAX_L2_TX_GAS_LIMIT, MAX_NEW_FACTORY_DEPS, U256, }; -use zksync_utils::h256_to_u256; +use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; pub(super) use self::result::SubmitTxError; use self::tx_sink::TxSink; @@ -630,6 +632,7 @@ impl TxSender { mut tx: Transaction, estimated_fee_scale_factor: f64, acceptable_overestimation: u32, + state_override: Option, ) -> Result { let estimation_started_at = Instant::now(); @@ -688,6 +691,140 @@ impl TxSender { ) })?; + if let Some(overrides) = &state_override { + for (account, overrides) in overrides { + if overrides.state.is_some() && overrides.state_diff.is_some() { + return Err(SubmitTxError::Unexecutable( + "state and stateDiff are mutually exclusive".to_string(), + )); + } + + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(account); + let log = StorageLog::new_write_log(balance_key, u256_to_h256(balance)); + self.acquire_replica_connection() + .await? + .storage_logs_dal() + .append_storage_logs( + block_args.resolved_block_number(), + &[(H256::zero(), vec![log])], + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(account); + let mut connection = self.acquire_replica_connection().await?; + + let full_nonce = connection + .storage_web3_dal() + .get_value(&nonce_key) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + + let log = StorageLog::new_write_log(nonce_key, u256_to_h256(new_full_nonce)); + connection + .storage_logs_dal() + .append_storage_logs( + block_args.resolved_block_number(), + &[(H256::zero(), vec![log])], + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + } + + if let Some(code) = &overrides.code { + let code_key = get_code_key(account); + let code_hash = hash_bytecode(&code.0); + + let mut connection = self.acquire_replica_connection().await?; + + let log = StorageLog::new_write_log(code_key, code_hash); + connection + .storage_logs_dal() + .append_storage_logs( + block_args.resolved_block_number(), + &[(H256::zero(), vec![log])], + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + connection + .factory_deps_dal() + .insert_factory_deps( + block_args.resolved_block_number(), + &HashMap::from_iter(vec![(code_hash, code.0.clone())]), + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + } + + if let Some(state) = &overrides.state { + let mut connection = self.acquire_replica_connection().await?; + + // First find storage keys to clear + let keys_to_clear = connection + .storage_logs_dal() + .dump_all_storage_logs_for_tests() + .await + .iter() + .filter(|log| &log.address == account) + .map(|log| log.key) + .unique() + .collect::>(); + + // Now clear keys and add new values + let mut logs: Vec = keys_to_clear + .iter() + .map(|key| { + StorageLog::new_write_log( + StorageKey::new(AccountTreeId::new(*account), *key), + H256::zero(), + ) + }) + .collect(); + logs.extend(state.iter().map(|(key, value)| { + StorageLog::new_write_log( + StorageKey::new(AccountTreeId::new(*account), *key), + *value, + ) + })); + + connection + .storage_logs_dal() + .append_storage_logs( + block_args.resolved_block_number(), + &[(H256::zero(), logs)], + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + } + + if let Some(state_diff) = &overrides.state_diff { + let logs = state_diff + .iter() + .map(|(key, value)| { + StorageLog::new_write_log( + StorageKey::new(AccountTreeId::new(*account), *key), + *value, + ) + }) + .collect(); + self.acquire_replica_connection() + .await? + .storage_logs_dal() + .append_storage_logs( + block_args.resolved_block_number(), + &[(H256::zero(), logs)], + ) + .await + .map_err(|err| SubmitTxError::Internal(err.into()))?; + } + } + } + if !tx.is_l1() && account_code_hash == H256::zero() && tx.execute.value > self.get_balance(&tx.initiator_account()).await? diff --git a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs index d3e0db24d626..8f54d3d1ad6e 100644 --- a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs @@ -1,7 +1,7 @@ use zksync_types::{ api::{ - Block, BlockId, BlockIdVariant, BlockNumber, Log, Transaction, TransactionId, - TransactionReceipt, TransactionVariant, + Block, BlockId, BlockIdVariant, BlockNumber, Log, StateOverride, Transaction, + TransactionId, TransactionReceipt, TransactionVariant, }, transaction_request::CallRequest, web3::types::{FeeHistory, Index, SyncState}, @@ -31,8 +31,13 @@ impl EthNamespaceServer for EthNamespace { .map_err(into_jsrpc_error) } - async fn estimate_gas(&self, req: CallRequest, block: Option) -> RpcResult { - self.estimate_gas_impl(req, block) + async fn estimate_gas( + &self, + req: CallRequest, + block: Option, + state_override: Option, + ) -> RpcResult { + self.estimate_gas_impl(req, block, state_override) .await .map_err(into_jsrpc_error) } diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index 0f3590a54fcc..4dd92d6a7ff5 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -1,8 +1,8 @@ use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ api::{ - BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, - TransactionVariant, + BlockId, BlockNumber, GetLogsFilter, StateOverride, Transaction, TransactionId, + TransactionReceipt, TransactionVariant, }, l2::{L2Tx, TransactionType}, transaction_request::CallRequest, @@ -102,6 +102,7 @@ impl EthNamespace { &self, request: CallRequest, _block: Option, + state_override: Option, ) -> Result { const METHOD_NAME: &str = "estimate_gas"; @@ -147,7 +148,12 @@ impl EthNamespace { let fee = self .state .tx_sender - .get_txs_fee_in_wei(tx.into(), scale_factor, acceptable_overestimation) + .get_txs_fee_in_wei( + tx.into(), + scale_factor, + acceptable_overestimation, + state_override, + ) .await .map_err(|err| err.into_web3_error(METHOD_NAME))?; method_latency.observe(); diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs index 231c8c6e0bd8..2166645cb865 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/zks.rs @@ -118,7 +118,7 @@ impl ZksNamespace { self.state .tx_sender - .get_txs_fee_in_wei(tx, scale_factor, acceptable_overestimation) + .get_txs_fee_in_wei(tx, scale_factor, acceptable_overestimation, None) .await .map_err(|err| err.into_web3_error(method_name)) } diff --git a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs index 22760c37feb1..0617cc0b6349 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs @@ -4,7 +4,10 @@ use std::sync::atomic::{AtomicU32, Ordering}; use multivm::interface::{ExecutionResult, VmRevertReason}; use zksync_types::{ - get_intrinsic_constants, transaction_request::CallRequest, L2ChainId, PackedEthSignature, U256, + api::{OverrideAccount, StateOverride}, + get_intrinsic_constants, + transaction_request::CallRequest, + L2ChainId, PackedEthSignature, U256, }; use zksync_utils::u256_to_h256; use zksync_web3_decl::namespaces::DebugNamespaceClient; @@ -436,7 +439,7 @@ impl HttpTest for EstimateGasTest { for threshold in [10_000, 50_000, 100_000, 1_000_000] { self.gas_limit_threshold.store(threshold, Ordering::Relaxed); let output = client - .estimate_gas(l2_transaction.clone().into(), None) + .estimate_gas(l2_transaction.clone().into(), None, None) .await?; assert!( output >= U256::from(threshold), @@ -461,10 +464,15 @@ impl HttpTest for EstimateGasTest { let mut call_request = CallRequest::from(l2_transaction); call_request.from = Some(SendRawTransactionTest::private_key_and_address().1); call_request.value = Some(1_000_000.into()); - client.estimate_gas(call_request.clone(), None).await?; + client + .estimate_gas(call_request.clone(), None, None) + .await?; call_request.value = Some(U256::max_value()); - let error = client.estimate_gas(call_request, None).await.unwrap_err(); + let error = client + .estimate_gas(call_request, None, None) + .await + .unwrap_err(); if let ClientError::Call(error) = error { let error_msg = error.message(); assert!( @@ -487,3 +495,104 @@ async fn estimate_gas_basics() { async fn estimate_gas_after_snapshot_recovery() { test_http_server(EstimateGasTest::new(true)).await; } + +#[derive(Debug)] +struct EstimateGasWithStateOverrideTest { + gas_limit_threshold: Arc, + snapshot_recovery: bool, +} + +impl EstimateGasWithStateOverrideTest { + fn new(snapshot_recovery: bool) -> Self { + Self { + gas_limit_threshold: Arc::default(), + snapshot_recovery, + } + } +} + +#[async_trait] +impl HttpTest for EstimateGasWithStateOverrideTest { + fn storage_initialization(&self) -> StorageInitialization { + let snapshot_recovery = self.snapshot_recovery; + SendRawTransactionTest { snapshot_recovery }.storage_initialization() + } + + fn transaction_executor(&self) -> MockTransactionExecutor { + let mut tx_executor = MockTransactionExecutor::default(); + let pending_block_number = if self.snapshot_recovery { + StorageInitialization::SNAPSHOT_RECOVERY_BLOCK + 2 + } else { + MiniblockNumber(1) + }; + let gas_limit_threshold = self.gas_limit_threshold.clone(); + tx_executor.set_call_responses(move |tx, block_args| { + assert_eq!(tx.execute.calldata(), [] as [u8; 0]); + assert_eq!(tx.nonce(), Some(Nonce(0))); + assert_eq!(block_args.resolved_block_number(), pending_block_number); + + let gas_limit_threshold = gas_limit_threshold.load(Ordering::SeqCst); + if tx.gas_limit() >= U256::from(gas_limit_threshold) { + ExecutionResult::Success { output: vec![] } + } else { + ExecutionResult::Revert { + output: VmRevertReason::VmError, + } + } + }); + tx_executor + } + + async fn test(&self, client: &HttpClient, pool: &ConnectionPool) -> anyhow::Result<()> { + // Transaction with balance override + let l2_transaction = create_l2_transaction(10, 100); + let mut call_request = CallRequest::from(l2_transaction); + call_request.from = Some(Address::random()); + call_request.value = Some(1_000_000.into()); + + let mut state_override = StateOverride::new(); + state_override.insert( + call_request.from.clone().unwrap(), + OverrideAccount { + balance: Some(U256::max_value()), + nonce: None, + code: None, + state: None, + state_diff: None, + }, + ); + + client + .estimate_gas(call_request.clone(), None, Some(state_override)) + .await?; + + // Transaction that should fail without balance override + let l2_transaction = create_l2_transaction(10, 100); + let mut call_request = CallRequest::from(l2_transaction); + call_request.from = Some(Address::random()); + call_request.value = Some(1_000_000.into()); + + let error = client + .estimate_gas(call_request.clone(), None, None) + .await + .unwrap_err(); + if let ClientError::Call(error) = error { + let error_msg = error.message(); + assert!( + error_msg + .to_lowercase() + .contains("insufficient balance for transfer"), + "{error_msg}" + ); + } else { + panic!("Unexpected error: {error:?}"); + } + + Ok(()) + } +} + +#[tokio::test] +async fn estimate_gas_with_state_override() { + test_http_server(EstimateGasWithStateOverrideTest::new(false)).await; +} From 77f8d21eb11917a04bb2fccb6a76cb254df47410 Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Tue, 5 Mar 2024 13:54:37 -0300 Subject: [PATCH 02/26] chore: run zk format --- prover/prover_fri/src/gpu_prover_job_processor.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/prover/prover_fri/src/gpu_prover_job_processor.rs b/prover/prover_fri/src/gpu_prover_job_processor.rs index 04bfcb580fbb..e758a0cbd584 100644 --- a/prover/prover_fri/src/gpu_prover_job_processor.rs +++ b/prover/prover_fri/src/gpu_prover_job_processor.rs @@ -3,8 +3,9 @@ pub mod gpu_prover { use std::{collections::HashMap, sync::Arc, time::Instant}; use anyhow::Context as _; - use shivini::gpu_proof_config::GpuProofConfig; - use shivini::{gpu_prove_from_external_witness_data, ProverContext}; + use shivini::{ + gpu_proof_config::GpuProofConfig, gpu_prove_from_external_witness_data, ProverContext, + }; use tokio::task::JoinHandle; use zksync_config::configs::{fri_prover_group::FriProverGroupConfig, FriProverConfig}; use zksync_dal::{fri_prover_dal::types::SocketAddress, ConnectionPool}; From e2ad0f4c1690bbe45e1b43268a170c7878b99122 Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Wed, 6 Mar 2024 17:12:15 -0300 Subject: [PATCH 03/26] fix: change state overrides to use storage_view --- core/lib/types/src/api/mod.rs | 63 +++++- .../src/api_server/execution_sandbox/apply.rs | 26 ++- .../api_server/execution_sandbox/execute.rs | 7 +- .../src/api_server/execution_sandbox/tests.rs | 1 + .../api_server/execution_sandbox/validate.rs | 1 + .../src/api_server/tx_sender/mod.rs | 182 +++--------------- .../web3/backend_jsonrpsee/namespaces/eth.rs | 21 +- .../src/api_server/web3/namespaces/eth.rs | 6 +- .../src/api_server/web3/tests/vm.rs | 10 +- 9 files changed, 150 insertions(+), 167 deletions(-) diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 79c64c7ebba4..ada1da5a84cd 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, convert::TryFrom, ops::Deref}; use chrono::{DateTime, Utc}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; @@ -737,7 +737,8 @@ pub struct Proof { } /// Collection of overridden accounts, useful for `eth_estimateGas`. -pub type StateOverride = HashMap; +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StateOverride(pub HashMap); /// Account override for `eth_estimateGas`. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -749,3 +750,61 @@ pub struct OverrideAccount { pub state: Option>, pub state_diff: Option>, } + +#[derive(Debug, Clone)] +pub struct ValidatedStateOverride(HashMap); + +impl ValidatedStateOverride { + pub fn get(&self, address: &Address) -> Option<&ValidatedOverrideAccount> { + self.0.get(address) + } +} + +impl Deref for ValidatedStateOverride { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl TryFrom for ValidatedStateOverride { + type Error = String; + + fn try_from(value: StateOverride) -> Result { + let mut validated = HashMap::new(); + for (address, account) in value.0 { + let validated_account = ValidatedOverrideAccount { + balance: account.balance, + nonce: account.nonce, + code: account.code, + state: match (account.state, account.state_diff) { + (Some(state), None) => Some(OverrideState::State(state)), + (None, Some(state_diff)) => Some(OverrideState::StateDiff(state_diff)), + (Some(_), Some(_)) => { + return Err(format!( + "Account {address} has overriden state and state_diff" + )) + } + (None, None) => None, + }, + }; + validated.insert(address, validated_account); + } + Ok(ValidatedStateOverride(validated)) + } +} + +#[derive(Debug, Clone)] +pub struct ValidatedOverrideAccount { + pub balance: Option, + pub nonce: Option, + pub code: Option, + pub state: Option, +} + +#[derive(Debug, Clone)] +pub enum OverrideState { + State(HashMap), + StateDiff(HashMap), +} diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index 147c8743bdb9..3d65b3c90107 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -23,7 +23,7 @@ use zksync_system_constants::{ SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, }; use zksync_types::{ - api, + api::{self, ValidatedStateOverride}, block::{pack_block_info, unpack_block_info, MiniblockHasher}, fee_model::BatchFeeInput, get_nonce_key, @@ -294,6 +294,7 @@ pub(super) fn apply_vm_in_sandbox( connection_pool: &ConnectionPool, tx: Transaction, block_args: BlockArgs, + state_override: Option, apply: impl FnOnce( &mut VmInstance>, HistoryDisabled>, Transaction, @@ -328,6 +329,29 @@ pub(super) fn apply_vm_in_sandbox( tx.initiator_account(), tx.nonce().unwrap_or(Nonce(0)) ); + + // Apply state override + if let Some(state_override) = state_override { + let mut storage_view = storage_view.as_ref().borrow_mut(); + + for (address, overrides) in state_override.iter() { + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(&address); + storage_view.set_value(balance_key, u256_to_h256(balance)); + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(&address); + + let full_nonce = storage_view.read_value(&nonce_key); + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + + storage_view.set_value(nonce_key, u256_to_h256(new_full_nonce)); + } + } + } + let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); let result = apply(&mut vm, tx); let vm_execution_took = execution_latency.observe(); diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs index f194ce3bbb1b..357595aeba89 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs @@ -10,8 +10,8 @@ use multivm::{ use tracing::{span, Level}; use zksync_dal::ConnectionPool; use zksync_types::{ - fee::TransactionExecutionMetrics, l2::L2Tx, ExecuteTransactionCommon, Nonce, - PackedEthSignature, Transaction, U256, + api::ValidatedStateOverride, fee::TransactionExecutionMetrics, l2::L2Tx, + ExecuteTransactionCommon, Nonce, PackedEthSignature, Transaction, U256, }; #[cfg(test)] @@ -111,6 +111,7 @@ impl TransactionExecutor { connection_pool: ConnectionPool, tx: Transaction, block_args: BlockArgs, + state_override: Option, custom_tracers: Vec, ) -> anyhow::Result { #[cfg(test)] @@ -134,6 +135,7 @@ impl TransactionExecutor { &connection_pool, tx, block_args, + state_override, |vm, tx| { let storage_invocation_tracer = StorageInvocations::new(execution_args.missed_storage_invocation_limit); @@ -196,6 +198,7 @@ impl TransactionExecutor { connection_pool, tx.into(), block_args, + None, custom_tracers, ) .await?; diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/tests.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/tests.rs index f66455ce0dca..0961e05363bf 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/tests.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/tests.rs @@ -187,6 +187,7 @@ async fn test_instantiating_vm(pool: ConnectionPool, block_args: BlockArgs) { &pool, transaction.clone(), block_args, + None, |_, received_tx| { assert_eq!(received_tx, transaction); }, diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/validate.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/validate.rs index 462977a8d36c..796179bf9a50 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/validate.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/validate.rs @@ -69,6 +69,7 @@ impl TransactionExecutor { &connection_pool, tx, block_args, + None, |vm, tx| { let stage_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Validation].start(); let span = tracing::debug_span!("validation").entered(); diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index 0bbeb4e16fd5..c1ed7876b872 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -1,9 +1,8 @@ //! Helper module to submit transactions into the zkSync Network. -use std::{cmp, collections::HashMap, sync::Arc, time::Instant}; +use std::{cmp, sync::Arc, time::Instant}; use anyhow::Context as _; -use itertools::Itertools; use multivm::{ interface::VmExecutionResultAndLogs, utils::{adjust_pubdata_price_for_tx, derive_base_fee_and_gas_per_pubdata, derive_overhead}, @@ -14,18 +13,18 @@ use zksync_contracts::BaseSystemContracts; use zksync_dal::{transactions_dal::L2TxSubmissionResult, ConnectionPool, StorageProcessor}; use zksync_state::PostgresStorageCaches; use zksync_types::{ - api::StateOverride, + api::ValidatedStateOverride, fee::{Fee, TransactionExecutionMetrics}, fee_model::BatchFeeInput, - get_code_key, get_intrinsic_constants, get_nonce_key, + get_code_key, get_intrinsic_constants, l1::is_l1_tx_type, l2::{error::TxCheckError::TxDuplication, L2Tx}, - utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, + utils::storage_key_for_eth_balance, AccountTreeId, Address, ExecuteTransactionCommon, L2ChainId, MiniblockNumber, Nonce, - PackedEthSignature, ProtocolVersionId, StorageKey, StorageLog, Transaction, VmVersion, H160, - H256, MAX_L2_TX_GAS_LIMIT, MAX_NEW_FACTORY_DEPS, U256, + PackedEthSignature, ProtocolVersionId, Transaction, VmVersion, H160, H256, MAX_L2_TX_GAS_LIMIT, + MAX_NEW_FACTORY_DEPS, U256, }; -use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; +use zksync_utils::h256_to_u256; pub(super) use self::result::SubmitTxError; use self::tx_sink::TxSink; @@ -304,6 +303,7 @@ impl TxSender { self.0.replica_connection_pool.clone(), tx.clone().into(), block_args, + None, vec![], ) .await?; @@ -561,6 +561,7 @@ impl TxSender { block_args: BlockArgs, base_fee: u64, vm_version: VmVersion, + state_override: Option, ) -> anyhow::Result<(VmExecutionResultAndLogs, TransactionExecutionMetrics)> { let gas_limit_with_overhead = tx_gas_limit + derive_overhead( @@ -606,6 +607,7 @@ impl TxSender { self.0.replica_connection_pool.clone(), tx.clone(), block_args, + state_override, vec![], ) .await?; @@ -631,7 +633,7 @@ impl TxSender { mut tx: Transaction, estimated_fee_scale_factor: f64, acceptable_overestimation: u32, - state_override: Option, + state_override: Option, ) -> Result { let estimation_started_at = Instant::now(); @@ -693,153 +695,27 @@ impl TxSender { ) })?; - if let Some(overrides) = &state_override { - for (account, overrides) in overrides { - if overrides.state.is_some() && overrides.state_diff.is_some() { - return Err(SubmitTxError::Unexecutable( - "state and stateDiff are mutually exclusive".to_string(), - )); - } - - if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(account); - let log = StorageLog::new_write_log(balance_key, u256_to_h256(balance)); - self.acquire_replica_connection() - .await? - .storage_logs_dal() - .append_storage_logs( - block_args.resolved_block_number(), - &[(H256::zero(), vec![log])], - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - } - - if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(account); - let mut connection = self.acquire_replica_connection().await?; - - let full_nonce = connection - .storage_web3_dal() - .get_value(&nonce_key) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); - let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - - let log = StorageLog::new_write_log(nonce_key, u256_to_h256(new_full_nonce)); - connection - .storage_logs_dal() - .append_storage_logs( - block_args.resolved_block_number(), - &[(H256::zero(), vec![log])], - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - } - - if let Some(code) = &overrides.code { - let code_key = get_code_key(account); - let code_hash = hash_bytecode(&code.0); - - let mut connection = self.acquire_replica_connection().await?; - - let log = StorageLog::new_write_log(code_key, code_hash); - connection - .storage_logs_dal() - .append_storage_logs( - block_args.resolved_block_number(), - &[(H256::zero(), vec![log])], - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - connection - .factory_deps_dal() - .insert_factory_deps( - block_args.resolved_block_number(), - &HashMap::from_iter(vec![(code_hash, code.0.clone())]), - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - } - - if let Some(state) = &overrides.state { - let mut connection = self.acquire_replica_connection().await?; - - // First find storage keys to clear - let keys_to_clear = connection - .storage_logs_dal() - .dump_all_storage_logs_for_tests() - .await - .iter() - .filter(|log| &log.address == account) - .map(|log| log.key) - .unique() - .collect::>(); - - // Now clear keys and add new values - let mut logs: Vec = keys_to_clear - .iter() - .map(|key| { - StorageLog::new_write_log( - StorageKey::new(AccountTreeId::new(*account), *key), - H256::zero(), - ) - }) - .collect(); - logs.extend(state.iter().map(|(key, value)| { - StorageLog::new_write_log( - StorageKey::new(AccountTreeId::new(*account), *key), - *value, - ) - })); - - connection - .storage_logs_dal() - .append_storage_logs( - block_args.resolved_block_number(), - &[(H256::zero(), logs)], - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - } + if !tx.is_l1() && account_code_hash == H256::zero() { + let balance = match state_override + .as_ref() + .and_then(|overrides| overrides.get(&tx.initiator_account())) + .and_then(|account| account.balance) + { + Some(balance) => balance.to_owned(), + None => self.get_balance(&tx.initiator_account()).await?, + }; - if let Some(state_diff) = &overrides.state_diff { - let logs = state_diff - .iter() - .map(|(key, value)| { - StorageLog::new_write_log( - StorageKey::new(AccountTreeId::new(*account), *key), - *value, - ) - }) - .collect(); - self.acquire_replica_connection() - .await? - .storage_logs_dal() - .append_storage_logs( - block_args.resolved_block_number(), - &[(H256::zero(), logs)], - ) - .await - .map_err(|err| SubmitTxError::Internal(err.into()))?; - } + if tx.execute.value > balance { + tracing::info!( + "fee estimation failed on validation step. + account: {} does not have enough funds for for transferring tx.value: {}.", + &tx.initiator_account(), + tx.execute.value + ); + return Err(SubmitTxError::InsufficientFundsForTransfer); } } - if !tx.is_l1() - && account_code_hash == H256::zero() - && tx.execute.value > self.get_balance(&tx.initiator_account()).await? - { - tracing::info!( - "fee estimation failed on validation step. - account: {} does not have enough funds for for transferring tx.value: {}.", - &tx.initiator_account(), - tx.execute.value - ); - return Err(SubmitTxError::InsufficientFundsForTransfer); - } - // For L2 transactions we need a properly formatted signature if let ExecuteTransactionCommon::L2(l2_common_data) = &mut tx.common_data { if l2_common_data.signature.is_empty() { @@ -906,6 +782,7 @@ impl TxSender { block_args, base_fee, protocol_version.into(), + state_override.clone(), ) .await .context("estimate_gas step failed")?; @@ -946,6 +823,7 @@ impl TxSender { block_args, base_fee, protocol_version.into(), + state_override, ) .await .context("final estimate_gas step failed")?; diff --git a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs index e9788ac2e74f..42ea121ba2db 100644 --- a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs @@ -1,14 +1,17 @@ use zksync_types::{ api::{ Block, BlockId, BlockIdVariant, BlockNumber, Log, StateOverride, Transaction, - TransactionId, TransactionReceipt, TransactionVariant, + TransactionId, TransactionReceipt, TransactionVariant, ValidatedStateOverride, }, transaction_request::CallRequest, web3::types::{FeeHistory, Index, SyncState}, Address, Bytes, H256, U256, U64, }; use zksync_web3_decl::{ - jsonrpsee::core::{async_trait, RpcResult}, + jsonrpsee::{ + core::{async_trait, RpcResult}, + types::error::{ErrorObjectOwned, INVALID_PARAMS_CODE}, + }, namespaces::eth::EthNamespaceServer, types::{Filter, FilterChanges}, }; @@ -39,7 +42,19 @@ impl EthNamespaceServer for EthNamespace { block: Option, state_override: Option, ) -> RpcResult { - self.estimate_gas_impl(req, block, state_override) + let validated_state_override = state_override + .map(|state_override| { + ValidatedStateOverride::try_from(state_override).map_err(|err| { + ErrorObjectOwned::owned( + INVALID_PARAMS_CODE, + format!("Invalid state override: {}", err), + None::<()>, + ) + }) + }) + .map_or(Ok(None), |res| res.map(Some))?; + + self.estimate_gas_impl(req, block, validated_state_override) .await .map_err(|err| self.current_method().map_err(err)) } diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index a3fb8e6f51e5..bd989c71f6d3 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -2,8 +2,8 @@ use anyhow::Context as _; use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ api::{ - BlockId, BlockNumber, GetLogsFilter, StateOverride, Transaction, TransactionId, - TransactionReceipt, TransactionVariant, + BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, + TransactionVariant, ValidatedStateOverride, }, l2::{L2Tx, TransactionType}, transaction_request::CallRequest, @@ -92,7 +92,7 @@ impl EthNamespace { &self, request: CallRequest, _block: Option, - state_override: Option, + state_override: Option, ) -> Result { let mut request_with_gas_per_pubdata_overridden = request; self.state diff --git a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs index 0617cc0b6349..fbbf7d5dabd1 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs @@ -543,16 +543,16 @@ impl HttpTest for EstimateGasWithStateOverrideTest { tx_executor } - async fn test(&self, client: &HttpClient, pool: &ConnectionPool) -> anyhow::Result<()> { + async fn test(&self, client: &HttpClient, _pool: &ConnectionPool) -> anyhow::Result<()> { // Transaction with balance override let l2_transaction = create_l2_transaction(10, 100); let mut call_request = CallRequest::from(l2_transaction); call_request.from = Some(Address::random()); call_request.value = Some(1_000_000.into()); - let mut state_override = StateOverride::new(); - state_override.insert( - call_request.from.clone().unwrap(), + let mut state_override_map = HashMap::new(); + state_override_map.insert( + call_request.from.unwrap(), OverrideAccount { balance: Some(U256::max_value()), nonce: None, @@ -561,6 +561,7 @@ impl HttpTest for EstimateGasWithStateOverrideTest { state_diff: None, }, ); + let state_override = StateOverride(state_override_map); client .estimate_gas(call_request.clone(), None, Some(state_override)) @@ -576,6 +577,7 @@ impl HttpTest for EstimateGasWithStateOverrideTest { .estimate_gas(call_request.clone(), None, None) .await .unwrap_err(); + if let ClientError::Call(error) = error { let error_msg = error.message(); assert!( From aa58304548629592f268fa1a96f03a769fc8f3c2 Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:32:19 -0300 Subject: [PATCH 04/26] feat: improve `StateOverride` with custom serializer --- core/lib/types/src/api/mod.rs | 87 ++++++++----------- .../src/api_server/execution_sandbox/apply.rs | 4 +- .../api_server/execution_sandbox/execute.rs | 6 +- .../src/api_server/tx_sender/mod.rs | 6 +- .../web3/backend_jsonrpsee/namespaces/eth.rs | 21 +---- .../src/api_server/web3/namespaces/eth.rs | 6 +- .../src/api_server/web3/tests/vm.rs | 1 - 7 files changed, 51 insertions(+), 80 deletions(-) diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index ada1da5a84cd..dbe40468bb6e 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, convert::TryFrom, ops::Deref}; +use std::{collections::HashMap, ops::Deref}; use chrono::{DateTime, Utc}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; @@ -747,64 +747,51 @@ pub struct OverrideAccount { pub balance: Option, pub nonce: Option, pub code: Option, - pub state: Option>, - pub state_diff: Option>, + #[serde(flatten, deserialize_with = "state_deserializer")] + pub state: Option, } -#[derive(Debug, Clone)] -pub struct ValidatedStateOverride(HashMap); - -impl ValidatedStateOverride { - pub fn get(&self, address: &Address) -> Option<&ValidatedOverrideAccount> { - self.0.get(address) - } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum OverrideState { + State(HashMap), + StateDiff(HashMap), } -impl Deref for ValidatedStateOverride { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.0 +fn state_deserializer<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let val = serde_json::Value::deserialize(deserializer)?; + let state: Option> = match val.get("state") { + Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, + None => None, + }; + let state_diff: Option> = match val.get("stateDiff") { + Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, + None => None, + }; + + match (state, state_diff) { + (Some(state), None) => Ok(Some(OverrideState::State(state))), + (None, Some(state_diff)) => Ok(Some(OverrideState::StateDiff(state_diff))), + (None, None) => Ok(None), + _ => Err(serde::de::Error::custom( + "Both 'state' and 'stateDiff' cannot be set simultaneously", + )), } } -impl TryFrom for ValidatedStateOverride { - type Error = String; - - fn try_from(value: StateOverride) -> Result { - let mut validated = HashMap::new(); - for (address, account) in value.0 { - let validated_account = ValidatedOverrideAccount { - balance: account.balance, - nonce: account.nonce, - code: account.code, - state: match (account.state, account.state_diff) { - (Some(state), None) => Some(OverrideState::State(state)), - (None, Some(state_diff)) => Some(OverrideState::StateDiff(state_diff)), - (Some(_), Some(_)) => { - return Err(format!( - "Account {address} has overriden state and state_diff" - )) - } - (None, None) => None, - }, - }; - validated.insert(address, validated_account); - } - Ok(ValidatedStateOverride(validated)) +impl StateOverride { + pub fn get(&self, address: &Address) -> Option<&OverrideAccount> { + self.0.get(address) } } -#[derive(Debug, Clone)] -pub struct ValidatedOverrideAccount { - pub balance: Option, - pub nonce: Option, - pub code: Option, - pub state: Option, -} +impl Deref for StateOverride { + type Target = HashMap; -#[derive(Debug, Clone)] -pub enum OverrideState { - State(HashMap), - StateDiff(HashMap), + fn deref(&self) -> &Self::Target { + &self.0 + } } diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index 3d65b3c90107..1e0d77b1c5de 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -23,7 +23,7 @@ use zksync_system_constants::{ SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, }; use zksync_types::{ - api::{self, ValidatedStateOverride}, + api::{self, StateOverride}, block::{pack_block_info, unpack_block_info, MiniblockHasher}, fee_model::BatchFeeInput, get_nonce_key, @@ -294,7 +294,7 @@ pub(super) fn apply_vm_in_sandbox( connection_pool: &ConnectionPool, tx: Transaction, block_args: BlockArgs, - state_override: Option, + state_override: Option, apply: impl FnOnce( &mut VmInstance>, HistoryDisabled>, Transaction, diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs index 357595aeba89..fd742a1b1dbc 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs @@ -10,8 +10,8 @@ use multivm::{ use tracing::{span, Level}; use zksync_dal::ConnectionPool; use zksync_types::{ - api::ValidatedStateOverride, fee::TransactionExecutionMetrics, l2::L2Tx, - ExecuteTransactionCommon, Nonce, PackedEthSignature, Transaction, U256, + api::StateOverride, fee::TransactionExecutionMetrics, l2::L2Tx, ExecuteTransactionCommon, + Nonce, PackedEthSignature, Transaction, U256, }; #[cfg(test)] @@ -111,7 +111,7 @@ impl TransactionExecutor { connection_pool: ConnectionPool, tx: Transaction, block_args: BlockArgs, - state_override: Option, + state_override: Option, custom_tracers: Vec, ) -> anyhow::Result { #[cfg(test)] diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index c1ed7876b872..c7366e35ee12 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -13,7 +13,7 @@ use zksync_contracts::BaseSystemContracts; use zksync_dal::{transactions_dal::L2TxSubmissionResult, ConnectionPool, StorageProcessor}; use zksync_state::PostgresStorageCaches; use zksync_types::{ - api::ValidatedStateOverride, + api::StateOverride, fee::{Fee, TransactionExecutionMetrics}, fee_model::BatchFeeInput, get_code_key, get_intrinsic_constants, @@ -561,7 +561,7 @@ impl TxSender { block_args: BlockArgs, base_fee: u64, vm_version: VmVersion, - state_override: Option, + state_override: Option, ) -> anyhow::Result<(VmExecutionResultAndLogs, TransactionExecutionMetrics)> { let gas_limit_with_overhead = tx_gas_limit + derive_overhead( @@ -633,7 +633,7 @@ impl TxSender { mut tx: Transaction, estimated_fee_scale_factor: f64, acceptable_overestimation: u32, - state_override: Option, + state_override: Option, ) -> Result { let estimation_started_at = Instant::now(); diff --git a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs index 42ea121ba2db..e9788ac2e74f 100644 --- a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs @@ -1,17 +1,14 @@ use zksync_types::{ api::{ Block, BlockId, BlockIdVariant, BlockNumber, Log, StateOverride, Transaction, - TransactionId, TransactionReceipt, TransactionVariant, ValidatedStateOverride, + TransactionId, TransactionReceipt, TransactionVariant, }, transaction_request::CallRequest, web3::types::{FeeHistory, Index, SyncState}, Address, Bytes, H256, U256, U64, }; use zksync_web3_decl::{ - jsonrpsee::{ - core::{async_trait, RpcResult}, - types::error::{ErrorObjectOwned, INVALID_PARAMS_CODE}, - }, + jsonrpsee::core::{async_trait, RpcResult}, namespaces::eth::EthNamespaceServer, types::{Filter, FilterChanges}, }; @@ -42,19 +39,7 @@ impl EthNamespaceServer for EthNamespace { block: Option, state_override: Option, ) -> RpcResult { - let validated_state_override = state_override - .map(|state_override| { - ValidatedStateOverride::try_from(state_override).map_err(|err| { - ErrorObjectOwned::owned( - INVALID_PARAMS_CODE, - format!("Invalid state override: {}", err), - None::<()>, - ) - }) - }) - .map_or(Ok(None), |res| res.map(Some))?; - - self.estimate_gas_impl(req, block, validated_state_override) + self.estimate_gas_impl(req, block, state_override) .await .map_err(|err| self.current_method().map_err(err)) } diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index bd989c71f6d3..a3fb8e6f51e5 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -2,8 +2,8 @@ use anyhow::Context as _; use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ api::{ - BlockId, BlockNumber, GetLogsFilter, Transaction, TransactionId, TransactionReceipt, - TransactionVariant, ValidatedStateOverride, + BlockId, BlockNumber, GetLogsFilter, StateOverride, Transaction, TransactionId, + TransactionReceipt, TransactionVariant, }, l2::{L2Tx, TransactionType}, transaction_request::CallRequest, @@ -92,7 +92,7 @@ impl EthNamespace { &self, request: CallRequest, _block: Option, - state_override: Option, + state_override: Option, ) -> Result { let mut request_with_gas_per_pubdata_overridden = request; self.state diff --git a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs index fbbf7d5dabd1..42717551b40e 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs @@ -558,7 +558,6 @@ impl HttpTest for EstimateGasWithStateOverrideTest { nonce: None, code: None, state: None, - state_diff: None, }, ); let state_override = StateOverride(state_override_map); From 0327102d39f3b5d5177973e1a730c92eee962bdb Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Thu, 7 Mar 2024 12:35:14 -0300 Subject: [PATCH 05/26] fix: remove pub from hashmap --- core/lib/types/src/api/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index dbe40468bb6e..9de604b9a1d4 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -738,7 +738,7 @@ pub struct Proof { /// Collection of overridden accounts, useful for `eth_estimateGas`. #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StateOverride(pub HashMap); +pub struct StateOverride(HashMap); /// Account override for `eth_estimateGas`. #[derive(Debug, Clone, Serialize, Deserialize)] From 9f9e3281afd59f824463fbdc5b8816da5a1f0620 Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Mon, 11 Mar 2024 12:25:17 -0300 Subject: [PATCH 06/26] feat: add storage overrides struct --- core/lib/state/src/lib.rs | 2 + core/lib/state/src/storage_overrides.rs | 85 +++++++++++++++++ core/lib/types/src/api/mod.rs | 4 + core/lib/types/src/transaction_request.rs | 4 +- core/lib/zksync_core/Cargo.toml | 1 + .../src/api_server/execution_sandbox/apply.rs | 93 +++++++++++++------ .../src/api_server/web3/tests/vm.rs | 46 ++++++++- 7 files changed, 203 insertions(+), 32 deletions(-) create mode 100644 core/lib/state/src/storage_overrides.rs diff --git a/core/lib/state/src/lib.rs b/core/lib/state/src/lib.rs index f76f38846566..991a8237e82f 100644 --- a/core/lib/state/src/lib.rs +++ b/core/lib/state/src/lib.rs @@ -22,6 +22,7 @@ mod in_memory; mod postgres; mod rocksdb; mod shadow_storage; +mod storage_overrides; mod storage_view; #[cfg(test)] mod test_utils; @@ -32,6 +33,7 @@ pub use self::{ postgres::{PostgresStorage, PostgresStorageCaches}, rocksdb::{RocksbStorageBuilder, RocksdbStorage}, shadow_storage::ShadowStorage, + storage_overrides::StorageOverrides, storage_view::{StorageView, StorageViewMetrics}, witness::WitnessStorage, }; diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs new file mode 100644 index 000000000000..6d87d2d87241 --- /dev/null +++ b/core/lib/state/src/storage_overrides.rs @@ -0,0 +1,85 @@ +use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; + +use zksync_types::{AccountTreeId, StorageKey, StorageValue, H256}; + +use crate::{ReadStorage, StorageView, StorageViewMetrics, WriteStorage}; + +/// A storage view that allows to override some of the storage values. +#[derive(Debug)] +pub struct StorageOverrides { + storage_view: StorageView, + overrided_factory_deps: HashMap>, + overrided_account_state: HashMap>, +} + +impl StorageOverrides { + /// Creates a new storage view based on the underlying storage. + pub fn new(storage_view: StorageView) -> Self { + Self { + storage_view, + overrided_factory_deps: HashMap::new(), + overrided_account_state: HashMap::new(), + } + } + + /// Overrides a factory dep code. + pub fn store_factory_dep(&mut self, hash: H256, code: Vec) { + self.overrided_factory_deps.insert(hash, code); + } + + /// Overrides an account entire state. + pub fn override_account_state(&mut self, account: AccountTreeId, state: HashMap) { + self.overrided_account_state.insert(account, state); + } + + /// Returns the current metrics. + pub fn metrics(&self) -> StorageViewMetrics { + self.storage_view.metrics() + } + + /// Make a Rc RefCell ptr to the storage + pub fn to_rc_ptr(self) -> Rc> { + Rc::new(RefCell::new(self)) + } +} + +impl ReadStorage for StorageOverrides { + fn read_value(&mut self, key: &StorageKey) -> StorageValue { + self.overrided_account_state + .get(key.account()) + .and_then(|f| f.get(key.key())) + .cloned() + .unwrap_or_else(|| self.storage_view.read_value(key)) + } + + /// Only keys contained in the underlying storage will return `false`. If a key was + /// inserted using [`Self::set_value()`], it will still return `true`. + fn is_write_initial(&mut self, key: &StorageKey) -> bool { + self.storage_view.is_write_initial(key) + } + + fn load_factory_dep(&mut self, hash: H256) -> Option> { + self.overrided_factory_deps + .get(&hash) + .cloned() + .or_else(|| self.storage_view.load_factory_dep(hash)) + } + + fn get_enumeration_index(&mut self, key: &StorageKey) -> Option { + self.storage_view.get_enumeration_index(key) + } +} + +impl WriteStorage for StorageOverrides { + fn set_value(&mut self, key: StorageKey, value: StorageValue) -> StorageValue { + self.storage_view.set_value(key, value) + } + + fn modified_storage_keys(&self) -> &HashMap { + self.storage_view.modified_storage_keys() + } + + fn missed_storage_invocations(&self) -> usize { + self.storage_view.missed_storage_invocations() + } +} diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 9de604b9a1d4..232a612d93b4 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -783,6 +783,10 @@ where } impl StateOverride { + pub fn new(state: HashMap) -> Self { + Self(state) + } + pub fn get(&self, address: &Address) -> Option<&OverrideAccount> { self.0.get(address) } diff --git a/core/lib/types/src/transaction_request.rs b/core/lib/types/src/transaction_request.rs index 01123191c097..820fc5eb3d23 100644 --- a/core/lib/types/src/transaction_request.rs +++ b/core/lib/types/src/transaction_request.rs @@ -384,7 +384,9 @@ impl TransactionRequest { } // returns packed eth signature if it is present - fn get_packed_signature(&self) -> Result { + pub fn get_packed_signature( + &self, + ) -> Result { let packed_v = self .v .ok_or(SerializationTransactionError::IncompleteSignature)? diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml index 53849b297407..71dbd3274485 100644 --- a/core/lib/zksync_core/Cargo.toml +++ b/core/lib/zksync_core/Cargo.toml @@ -61,6 +61,7 @@ itertools = "0.10.3" metrics = "0.21" ctrlc = { version = "3.1", features = ["termination"] } rand = "0.8" +rlp = "0.5" tokio = { version = "1", features = ["time"] } futures = { version = "0.3", features = ["compat"] } diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index 1e0d77b1c5de..a4c33f0550dd 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -17,28 +17,32 @@ use multivm::{ }; use tokio::runtime::Handle; use zksync_dal::{ConnectionPool, StorageProcessor}; -use zksync_state::{PostgresStorage, ReadStorage, StoragePtr, StorageView, WriteStorage}; +use zksync_state::{ + PostgresStorage, ReadStorage, StorageOverrides, StoragePtr, StorageView, WriteStorage, +}; use zksync_system_constants::{ SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_CURRENT_L2_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, }; use zksync_types::{ - api::{self, StateOverride}, + api::{self, OverrideState, StateOverride}, block::{pack_block_info, unpack_block_info, MiniblockHasher}, fee_model::BatchFeeInput, - get_nonce_key, + get_code_key, get_nonce_key, utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, AccountTreeId, L1BatchNumber, MiniblockNumber, Nonce, ProtocolVersionId, StorageKey, Transaction, H256, U256, }; -use zksync_utils::{h256_to_u256, time::seconds_since_epoch, u256_to_h256}; +use zksync_utils::{ + bytecode::hash_bytecode, h256_to_u256, time::seconds_since_epoch, u256_to_h256, +}; use super::{ vm_metrics::{self, SandboxStage, SANDBOX_METRICS}, BlockArgs, TxExecutionArgs, TxSharedArgs, VmPermit, }; -type BoxedVm<'a> = Box>, HistoryDisabled>>; +type BoxedVm<'a> = Box>, HistoryDisabled>>; #[derive(Debug)] struct Sandbox<'a> { @@ -259,7 +263,10 @@ impl<'a> Sandbox<'a> { mut self, tx: &Transaction, adjust_pubdata_price: bool, - ) -> (BoxedVm<'a>, StoragePtr>>) { + ) -> ( + BoxedVm<'a>, + StoragePtr>>, + ) { self.setup_storage_view(tx); let protocol_version = self.system_env.version; if adjust_pubdata_price { @@ -271,14 +278,14 @@ impl<'a> Sandbox<'a> { ); }; - let storage_view = self.storage_view.to_rc_ptr(); + let storage_overrides = StorageOverrides::new(self.storage_view).to_rc_ptr(); let vm = Box::new(VmInstance::new_with_specific_version( self.l1_batch_env, self.system_env, - storage_view.clone(), + storage_overrides.clone(), protocol_version.into_api_vm_version(), )); - (vm, storage_view) + (vm, storage_overrides) } } @@ -296,7 +303,7 @@ pub(super) fn apply_vm_in_sandbox( block_args: BlockArgs, state_override: Option, apply: impl FnOnce( - &mut VmInstance>, HistoryDisabled>, + &mut VmInstance>, HistoryDisabled>, Transaction, ) -> T, ) -> anyhow::Result { @@ -332,24 +339,8 @@ pub(super) fn apply_vm_in_sandbox( // Apply state override if let Some(state_override) = state_override { - let mut storage_view = storage_view.as_ref().borrow_mut(); - - for (address, overrides) in state_override.iter() { - if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(&address); - storage_view.set_value(balance_key, u256_to_h256(balance)); - } - - if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(&address); - - let full_nonce = storage_view.read_value(&nonce_key); - let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); - let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - - storage_view.set_value(nonce_key, u256_to_h256(new_full_nonce)); - } - } + let mut storage_override = storage_view.as_ref().borrow_mut(); + apply_state_override(&state_override, &mut storage_override); } let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); @@ -521,3 +512,49 @@ impl BlockArgs { }) } } + +fn apply_state_override( + state_override: &StateOverride, + storage_override: &mut StorageOverrides, +) { + for (account, overrides) in state_override.iter() { + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(&account); + storage_override.set_value(balance_key, u256_to_h256(balance)); + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(&account); + + let full_nonce = storage_override.read_value(&nonce_key); + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + + storage_override.set_value(nonce_key, u256_to_h256(new_full_nonce)); + } + + if let Some(code) = &overrides.code { + let code_key = get_code_key(account); + let code_hash = hash_bytecode(&code.0); + + storage_override.set_value(code_key, code_hash); + storage_override.store_factory_dep(code_hash, code.0.clone()); + } + + match &overrides.state { + Some(OverrideState::State(state)) => { + storage_override + .override_account_state(AccountTreeId::new(account.clone()), state.clone()); + } + Some(OverrideState::StateDiff(state_diff)) => { + for (key, value) in state_diff.iter() { + storage_override.set_value( + StorageKey::new(AccountTreeId::new(account.clone()), key.clone()), + value.clone(), + ); + } + } + None => {} + } + } +} diff --git a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs index 42717551b40e..de87aed903f6 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs @@ -3,11 +3,14 @@ use std::sync::atomic::{AtomicU32, Ordering}; use multivm::interface::{ExecutionResult, VmRevertReason}; +use rlp::RlpStream; +use zksync_contracts::read_bytecode; +use zksync_test_account::{Account, TxType}; use zksync_types::{ - api::{OverrideAccount, StateOverride}, + api::{OverrideAccount, StateOverride, TransactionRequest}, get_intrinsic_constants, transaction_request::CallRequest, - L2ChainId, PackedEthSignature, U256, + Bytes, L2ChainId, PackedEthSignature, U256, }; use zksync_utils::u256_to_h256; use zksync_web3_decl::namespaces::DebugNamespaceClient; @@ -560,7 +563,7 @@ impl HttpTest for EstimateGasWithStateOverrideTest { state: None, }, ); - let state_override = StateOverride(state_override_map); + let state_override = StateOverride::new(state_override_map); client .estimate_gas(call_request.clone(), None, Some(state_override)) @@ -589,6 +592,43 @@ impl HttpTest for EstimateGasWithStateOverrideTest { panic!("Unexpected error: {error:?}"); } + // Test code hash override + let mut account = Account::random(); + let chain_id = client.chain_id().await.unwrap(); + + // deploy erc20 smart contract + let erc20_contract = read_bytecode( + "etc/contracts-test-data/artifacts-zk/contracts/erc20/erc20.sol/Erc20.json", + ); + + let deploy_tx = account.get_deploy_tx(&erc20_contract, None, TxType::L2); + let tx = deploy_tx.tx; + + let l2_tx = L2Tx::try_from(tx.clone()).unwrap(); + + let mut rlp_stream = RlpStream::new(); + let tx_request = TransactionRequest::from(l2_tx.clone()); + tx_request.rlp( + &mut rlp_stream, + chain_id.as_u64(), + Some(&tx_request.get_packed_signature()?), + ); + + let mut raw_tx = Bytes(rlp_stream.as_raw().to_vec()); + + // append transaction type at beginning + raw_tx.0.insert(0, tx.tx_format() as u8); + + client.send_raw_transaction(raw_tx).await?; + + // FIX: + // - Deploying erc20 contract fails with "Insufficient funds" + // TODO: + // - Try to call increment function + // - Call should fail as contract is erc20 + // - Change codehash to counter contract + // - Increment function now correctly works + Ok(()) } } From df873703611e99164af2a93ef14b510ffc7e6427 Mon Sep 17 00:00:00 2001 From: aon <21188659+aon@users.noreply.github.com> Date: Mon, 11 Mar 2024 13:10:54 -0300 Subject: [PATCH 07/26] fix: read value on overrided state --- core/lib/state/src/storage_overrides.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 6d87d2d87241..0e592780e8bd 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -45,10 +45,14 @@ impl StorageOverrides { impl ReadStorage for StorageOverrides { fn read_value(&mut self, key: &StorageKey) -> StorageValue { + // If the account is overrided, return the overrided value if any or zero. + // Otherwise, return the value from the underlying storage. self.overrided_account_state .get(key.account()) - .and_then(|f| f.get(key.key())) - .cloned() + .and_then(|state| match state.get(key.key()) { + Some(v) => Some(v.clone()), + None => Some(H256::zero()), + }) .unwrap_or_else(|| self.storage_view.read_value(key)) } From d1bee529a2002aaf47e6fce39309395b9b555ca4 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Tue, 12 Mar 2024 13:50:01 -0300 Subject: [PATCH 08/26] minor changes --- core/lib/state/src/storage_overrides.rs | 58 ++++++++++++++++- .../src/api_server/execution_sandbox/apply.rs | 65 ++++--------------- 2 files changed, 68 insertions(+), 55 deletions(-) diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 0e592780e8bd..64f3dec7da49 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -1,6 +1,12 @@ use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; -use zksync_types::{AccountTreeId, StorageKey, StorageValue, H256}; +use zksync_types::{ + api::{OverrideState, StateOverride}, + get_code_key, get_nonce_key, + utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, + AccountTreeId, StorageKey, StorageValue, H256, +}; +use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; use crate::{ReadStorage, StorageView, StorageViewMetrics, WriteStorage}; @@ -11,6 +17,50 @@ pub struct StorageOverrides { overrided_factory_deps: HashMap>, overrided_account_state: HashMap>, } +impl StorageOverrides { + /// Applies the state override to the storage. + pub fn apply_state_override(&mut self, state_override: StateOverride) { + for (account, overrides) in state_override.iter() { + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(&account); + self.set_value(balance_key, u256_to_h256(balance)); + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(&account); + + let full_nonce = self.read_value(&nonce_key); + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + + self.set_value(nonce_key, u256_to_h256(new_full_nonce)); + } + + if let Some(code) = &overrides.code { + let code_key = get_code_key(account); + let code_hash = hash_bytecode(&code.0); + + self.set_value(code_key, code_hash); + self.store_factory_dep(code_hash, code.0.clone()); + } + + match &overrides.state { + Some(OverrideState::State(state)) => { + self.override_account_state(AccountTreeId::new(account.clone()), state.clone()); + } + Some(OverrideState::StateDiff(state_diff)) => { + for (key, value) in state_diff.iter() { + self.set_value( + StorageKey::new(AccountTreeId::new(account.clone()), key.clone()), + value.clone(), + ); + } + } + None => {} + } + } + } +} impl StorageOverrides { /// Creates a new storage view based on the underlying storage. @@ -43,6 +93,12 @@ impl StorageOverrides { } } +impl Into>> for StorageOverrides { + fn into(self) -> Rc> { + self.to_rc_ptr() + } +} + impl ReadStorage for StorageOverrides { fn read_value(&mut self, key: &StorageKey) -> StorageValue { // If the account is overrided, return the overrided value if any or zero. diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index a4c33f0550dd..3fb058e08dab 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -6,7 +6,11 @@ //! //! This module is intended to be blocking. -use std::time::{Duration, Instant}; +use std::{ + cell::RefCell, + rc::Rc, + time::{Duration, Instant}, +}; use anyhow::Context as _; use multivm::{ @@ -25,17 +29,15 @@ use zksync_system_constants::{ SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, }; use zksync_types::{ - api::{self, OverrideState, StateOverride}, + api::{self, StateOverride}, block::{pack_block_info, unpack_block_info, MiniblockHasher}, fee_model::BatchFeeInput, - get_code_key, get_nonce_key, + get_nonce_key, utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, AccountTreeId, L1BatchNumber, MiniblockNumber, Nonce, ProtocolVersionId, StorageKey, Transaction, H256, U256, }; -use zksync_utils::{ - bytecode::hash_bytecode, h256_to_u256, time::seconds_since_epoch, u256_to_h256, -}; +use zksync_utils::{h256_to_u256, time::seconds_since_epoch, u256_to_h256}; use super::{ vm_metrics::{self, SandboxStage, SANDBOX_METRICS}, @@ -278,7 +280,8 @@ impl<'a> Sandbox<'a> { ); }; - let storage_overrides = StorageOverrides::new(self.storage_view).to_rc_ptr(); + let storage_overrides: Rc>>> = + StorageOverrides::new(self.storage_view).into(); let vm = Box::new(VmInstance::new_with_specific_version( self.l1_batch_env, self.system_env, @@ -340,7 +343,7 @@ pub(super) fn apply_vm_in_sandbox( // Apply state override if let Some(state_override) = state_override { let mut storage_override = storage_view.as_ref().borrow_mut(); - apply_state_override(&state_override, &mut storage_override); + storage_override.apply_state_override(state_override); } let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); @@ -512,49 +515,3 @@ impl BlockArgs { }) } } - -fn apply_state_override( - state_override: &StateOverride, - storage_override: &mut StorageOverrides, -) { - for (account, overrides) in state_override.iter() { - if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(&account); - storage_override.set_value(balance_key, u256_to_h256(balance)); - } - - if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(&account); - - let full_nonce = storage_override.read_value(&nonce_key); - let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); - let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - - storage_override.set_value(nonce_key, u256_to_h256(new_full_nonce)); - } - - if let Some(code) = &overrides.code { - let code_key = get_code_key(account); - let code_hash = hash_bytecode(&code.0); - - storage_override.set_value(code_key, code_hash); - storage_override.store_factory_dep(code_hash, code.0.clone()); - } - - match &overrides.state { - Some(OverrideState::State(state)) => { - storage_override - .override_account_state(AccountTreeId::new(account.clone()), state.clone()); - } - Some(OverrideState::StateDiff(state_diff)) => { - for (key, value) in state_diff.iter() { - storage_override.set_value( - StorageKey::new(AccountTreeId::new(account.clone()), key.clone()), - value.clone(), - ); - } - } - None => {} - } - } -} From 671689cd08911d8df752a313021ff66313719cdc Mon Sep 17 00:00:00 2001 From: Jrigada Date: Wed, 13 Mar 2024 13:39:43 -0300 Subject: [PATCH 09/26] create new state_override file and add tests --- core/lib/state/src/storage_overrides.rs | 5 +- core/lib/types/src/api/mod.rs | 67 +----------------- core/lib/types/src/api/state_override.rs | 70 +++++++++++++++++++ core/lib/web3_decl/src/namespaces/eth.rs | 5 +- .../src/api_server/execution_sandbox/apply.rs | 2 +- .../api_server/execution_sandbox/execute.rs | 4 +- .../src/api_server/tx_sender/mod.rs | 2 +- .../web3/backend_jsonrpsee/namespaces/eth.rs | 4 +- .../src/api_server/web3/namespaces/eth.rs | 4 +- .../src/api_server/web3/tests/vm.rs | 45 +----------- .../ts-integration/tests/api/web3.test.ts | 54 ++++++++++++++ 11 files changed, 142 insertions(+), 120 deletions(-) create mode 100644 core/lib/types/src/api/state_override.rs diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 64f3dec7da49..7f0e3fd3f817 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -1,7 +1,7 @@ use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; use zksync_types::{ - api::{OverrideState, StateOverride}, + api::state_override::{OverrideState, StateOverride}, get_code_key, get_nonce_key, utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, AccountTreeId, StorageKey, StorageValue, H256, @@ -17,6 +17,7 @@ pub struct StorageOverrides { overrided_factory_deps: HashMap>, overrided_account_state: HashMap>, } + impl StorageOverrides { /// Applies the state override to the storage. pub fn apply_state_override(&mut self, state_override: StateOverride) { @@ -37,7 +38,7 @@ impl StorageOverrides { } if let Some(code) = &overrides.code { - let code_key = get_code_key(account); + let code_key = get_code_key(&account); let code_hash = hash_bytecode(&code.0); self.set_value(code_key, code_hash); diff --git a/core/lib/types/src/api/mod.rs b/core/lib/types/src/api/mod.rs index 232a612d93b4..e8deddd54155 100644 --- a/core/lib/types/src/api/mod.rs +++ b/core/lib/types/src/api/mod.rs @@ -1,5 +1,3 @@ -use std::{collections::HashMap, ops::Deref}; - use chrono::{DateTime, Utc}; use serde::{de, Deserialize, Deserializer, Serialize, Serializer}; use strum::Display; @@ -20,6 +18,7 @@ use crate::{ }; pub mod en; +pub mod state_override; /// Block Number #[derive(Copy, Clone, Debug, PartialEq, Display)] @@ -735,67 +734,3 @@ pub struct Proof { pub address: Address, pub storage_proof: Vec, } - -/// Collection of overridden accounts, useful for `eth_estimateGas`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct StateOverride(HashMap); - -/// Account override for `eth_estimateGas`. -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub struct OverrideAccount { - pub balance: Option, - pub nonce: Option, - pub code: Option, - #[serde(flatten, deserialize_with = "state_deserializer")] - pub state: Option, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -pub enum OverrideState { - State(HashMap), - StateDiff(HashMap), -} - -fn state_deserializer<'de, D>(deserializer: D) -> Result, D::Error> -where - D: Deserializer<'de>, -{ - let val = serde_json::Value::deserialize(deserializer)?; - let state: Option> = match val.get("state") { - Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, - None => None, - }; - let state_diff: Option> = match val.get("stateDiff") { - Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, - None => None, - }; - - match (state, state_diff) { - (Some(state), None) => Ok(Some(OverrideState::State(state))), - (None, Some(state_diff)) => Ok(Some(OverrideState::StateDiff(state_diff))), - (None, None) => Ok(None), - _ => Err(serde::de::Error::custom( - "Both 'state' and 'stateDiff' cannot be set simultaneously", - )), - } -} - -impl StateOverride { - pub fn new(state: HashMap) -> Self { - Self(state) - } - - pub fn get(&self, address: &Address) -> Option<&OverrideAccount> { - self.0.get(address) - } -} - -impl Deref for StateOverride { - type Target = HashMap; - - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/core/lib/types/src/api/state_override.rs b/core/lib/types/src/api/state_override.rs new file mode 100644 index 000000000000..89d0d2d0d173 --- /dev/null +++ b/core/lib/types/src/api/state_override.rs @@ -0,0 +1,70 @@ +use std::{collections::HashMap, ops::Deref}; + +use serde::{Deserialize, Deserializer, Serialize}; +use zksync_basic_types::web3::types::{Bytes, H256, U256}; + +use crate::Address; + +/// Collection of overridden accounts, useful for `eth_estimateGas`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StateOverride(HashMap); + +/// Account override for `eth_estimateGas`. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct OverrideAccount { + pub balance: Option, + pub nonce: Option, + pub code: Option, + #[serde(flatten, deserialize_with = "state_deserializer")] + pub state: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum OverrideState { + State(HashMap), + StateDiff(HashMap), +} + +fn state_deserializer<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let val = serde_json::Value::deserialize(deserializer)?; + let state: Option> = match val.get("state") { + Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, + None => None, + }; + let state_diff: Option> = match val.get("stateDiff") { + Some(val) => serde_json::from_value(val.clone()).map_err(serde::de::Error::custom)?, + None => None, + }; + + match (state, state_diff) { + (Some(state), None) => Ok(Some(OverrideState::State(state))), + (None, Some(state_diff)) => Ok(Some(OverrideState::StateDiff(state_diff))), + (None, None) => Ok(None), + _ => Err(serde::de::Error::custom( + "Both 'state' and 'stateDiff' cannot be set simultaneously", + )), + } +} + +impl StateOverride { + pub fn new(state: HashMap) -> Self { + Self(state) + } + + pub fn get(&self, address: &Address) -> Option<&OverrideAccount> { + self.0.get(address) + } +} + +impl Deref for StateOverride { + type Target = HashMap; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/core/lib/web3_decl/src/namespaces/eth.rs b/core/lib/web3_decl/src/namespaces/eth.rs index eabbdb0bea00..389c40df1012 100644 --- a/core/lib/web3_decl/src/namespaces/eth.rs +++ b/core/lib/web3_decl/src/namespaces/eth.rs @@ -3,7 +3,10 @@ use jsonrpsee::{ proc_macros::rpc, }; use zksync_types::{ - api::{BlockId, BlockIdVariant, BlockNumber, StateOverride, Transaction, TransactionVariant}, + api::{ + state_override::StateOverride, BlockId, BlockIdVariant, BlockNumber, Transaction, + TransactionVariant, + }, transaction_request::CallRequest, Address, H256, }; diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index 3fb058e08dab..f968f8d5af0f 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -29,7 +29,7 @@ use zksync_system_constants::{ SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, }; use zksync_types::{ - api::{self, StateOverride}, + api::{self, state_override::StateOverride}, block::{pack_block_info, unpack_block_info, MiniblockHasher}, fee_model::BatchFeeInput, get_nonce_key, diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs index fd742a1b1dbc..90422f99573e 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/execute.rs @@ -10,8 +10,8 @@ use multivm::{ use tracing::{span, Level}; use zksync_dal::ConnectionPool; use zksync_types::{ - api::StateOverride, fee::TransactionExecutionMetrics, l2::L2Tx, ExecuteTransactionCommon, - Nonce, PackedEthSignature, Transaction, U256, + api::state_override::StateOverride, fee::TransactionExecutionMetrics, l2::L2Tx, + ExecuteTransactionCommon, Nonce, PackedEthSignature, Transaction, U256, }; #[cfg(test)] diff --git a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs index c7366e35ee12..92cdbd718fbb 100644 --- a/core/lib/zksync_core/src/api_server/tx_sender/mod.rs +++ b/core/lib/zksync_core/src/api_server/tx_sender/mod.rs @@ -13,7 +13,7 @@ use zksync_contracts::BaseSystemContracts; use zksync_dal::{transactions_dal::L2TxSubmissionResult, ConnectionPool, StorageProcessor}; use zksync_state::PostgresStorageCaches; use zksync_types::{ - api::StateOverride, + api::state_override::StateOverride, fee::{Fee, TransactionExecutionMetrics}, fee_model::BatchFeeInput, get_code_key, get_intrinsic_constants, diff --git a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs index e9788ac2e74f..3e393194f812 100644 --- a/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/backend_jsonrpsee/namespaces/eth.rs @@ -1,7 +1,7 @@ use zksync_types::{ api::{ - Block, BlockId, BlockIdVariant, BlockNumber, Log, StateOverride, Transaction, - TransactionId, TransactionReceipt, TransactionVariant, + state_override::StateOverride, Block, BlockId, BlockIdVariant, BlockNumber, Log, + Transaction, TransactionId, TransactionReceipt, TransactionVariant, }, transaction_request::CallRequest, web3::types::{FeeHistory, Index, SyncState}, diff --git a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs index a3fb8e6f51e5..1f89c9249c1a 100644 --- a/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs +++ b/core/lib/zksync_core/src/api_server/web3/namespaces/eth.rs @@ -2,8 +2,8 @@ use anyhow::Context as _; use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ api::{ - BlockId, BlockNumber, GetLogsFilter, StateOverride, Transaction, TransactionId, - TransactionReceipt, TransactionVariant, + state_override::StateOverride, BlockId, BlockNumber, GetLogsFilter, Transaction, + TransactionId, TransactionReceipt, TransactionVariant, }, l2::{L2Tx, TransactionType}, transaction_request::CallRequest, diff --git a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs index de87aed903f6..c85871aa9376 100644 --- a/core/lib/zksync_core/src/api_server/web3/tests/vm.rs +++ b/core/lib/zksync_core/src/api_server/web3/tests/vm.rs @@ -3,14 +3,11 @@ use std::sync::atomic::{AtomicU32, Ordering}; use multivm::interface::{ExecutionResult, VmRevertReason}; -use rlp::RlpStream; -use zksync_contracts::read_bytecode; -use zksync_test_account::{Account, TxType}; use zksync_types::{ - api::{OverrideAccount, StateOverride, TransactionRequest}, + api::state_override::{OverrideAccount, StateOverride}, get_intrinsic_constants, transaction_request::CallRequest, - Bytes, L2ChainId, PackedEthSignature, U256, + L2ChainId, PackedEthSignature, U256, }; use zksync_utils::u256_to_h256; use zksync_web3_decl::namespaces::DebugNamespaceClient; @@ -591,44 +588,6 @@ impl HttpTest for EstimateGasWithStateOverrideTest { } else { panic!("Unexpected error: {error:?}"); } - - // Test code hash override - let mut account = Account::random(); - let chain_id = client.chain_id().await.unwrap(); - - // deploy erc20 smart contract - let erc20_contract = read_bytecode( - "etc/contracts-test-data/artifacts-zk/contracts/erc20/erc20.sol/Erc20.json", - ); - - let deploy_tx = account.get_deploy_tx(&erc20_contract, None, TxType::L2); - let tx = deploy_tx.tx; - - let l2_tx = L2Tx::try_from(tx.clone()).unwrap(); - - let mut rlp_stream = RlpStream::new(); - let tx_request = TransactionRequest::from(l2_tx.clone()); - tx_request.rlp( - &mut rlp_stream, - chain_id.as_u64(), - Some(&tx_request.get_packed_signature()?), - ); - - let mut raw_tx = Bytes(rlp_stream.as_raw().to_vec()); - - // append transaction type at beginning - raw_tx.0.insert(0, tx.tx_format() as u8); - - client.send_raw_transaction(raw_tx).await?; - - // FIX: - // - Deploying erc20 contract fails with "Insufficient funds" - // TODO: - // - Try to call increment function - // - Call should fail as contract is erc20 - // - Change codehash to counter contract - // - Increment function now correctly works - Ok(()) } } diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index 8cdde989a9e3..d5dcb8e6aa2c 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -881,6 +881,60 @@ describe('web3 API compatibility tests', () => { expect(txFromApi.v! <= 1).toEqual(true); }); + describe('Storage override', () => { + test('Should be able to estimate_gas overriding the balance of the sender', async () => { + const balance = await alice.getBalance(l2Token); + const amount = balance.add(1); + // const tx = await alice.transfer({ to: alice.address, token: l2Token, amount }); + // await expect(tx).toBeRejected('insufficient balance for transfer'); + + // Call estimate_gas overriding the balance of the sender using the eth_estimateGas endpoint + const response = await alice.provider.send('eth_estimateGas', [ + { + from: alice.address, + to: alice.address, + value: amount.toHexString() + }, + 'latest', + //override with maximum balance + { [alice.address]: { balance: '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' } } + ]); + + // Assert that the response is successful + expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); + }); + test('Should be able to estimate_gas overriding contract code', async () => { + // Deploy the first contract + const contract1 = await deployContract(alice, contracts.events, []); + + // Deploy the second contract to extract the code that we are overriding the estimation with + const contract2 = await deployContract(alice, contracts.counter, []); + + // Get the code of contract2 + const code = await alice.provider.getCode(contract2.address); + + // Get the calldata of the increment function of contract2 + const incrementFunctionData = contract2.interface.encodeFunctionData('increment', [1]); + + // Assert that the estimation fails because the increment function is not present in contract1 + expect(alice.provider.estimateGas({ to: contract1.address, data: incrementFunctionData })).toBeRejected(); + + // Call estimate_gas overriding the code of contract1 with the code of contract2 using the eth_estimateGas endpoint + const response = await alice.provider.send('eth_estimateGas', [ + { + from: alice.address, + to: contract1.address, + data: incrementFunctionData + }, + 'latest', + { [contract1.address]: { code: code } } + ]); + + // Assert that the response is successful + expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); + }); + }); + afterAll(async () => { await testMaster.deinitialize(); }); From a640cee4b0587f9be7f6942eee85e6c964fcbab3 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 14 Mar 2024 14:24:05 -0300 Subject: [PATCH 10/26] linter and spellcheck --- core/lib/state/src/storage_overrides.rs | 38 ++++++++----------- .../src/api_server/execution_sandbox/apply.rs | 2 +- 2 files changed, 17 insertions(+), 23 deletions(-) diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 7f0e3fd3f817..db4b3b050a62 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -20,15 +20,15 @@ pub struct StorageOverrides { impl StorageOverrides { /// Applies the state override to the storage. - pub fn apply_state_override(&mut self, state_override: StateOverride) { + pub fn apply_state_override(&mut self, state_override: &StateOverride) { for (account, overrides) in state_override.iter() { if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(&account); + let balance_key = storage_key_for_eth_balance(account); self.set_value(balance_key, u256_to_h256(balance)); } if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(&account); + let nonce_key = get_nonce_key(account); let full_nonce = self.read_value(&nonce_key); let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); @@ -38,7 +38,7 @@ impl StorageOverrides { } if let Some(code) = &overrides.code { - let code_key = get_code_key(&account); + let code_key = get_code_key(account); let code_hash = hash_bytecode(&code.0); self.set_value(code_key, code_hash); @@ -47,14 +47,11 @@ impl StorageOverrides { match &overrides.state { Some(OverrideState::State(state)) => { - self.override_account_state(AccountTreeId::new(account.clone()), state.clone()); + self.override_account_state(AccountTreeId::new(*account), state.clone()); } Some(OverrideState::StateDiff(state_diff)) => { - for (key, value) in state_diff.iter() { - self.set_value( - StorageKey::new(AccountTreeId::new(account.clone()), key.clone()), - value.clone(), - ); + for (key, value) in state_diff { + self.set_value(StorageKey::new(AccountTreeId::new(*account), *key), *value); } } None => {} @@ -73,7 +70,7 @@ impl StorageOverrides { } } - /// Overrides a factory dep code. + /// Overrides a factory dependency code. pub fn store_factory_dep(&mut self, hash: H256, code: Vec) { self.overrided_factory_deps.insert(hash, code); } @@ -94,23 +91,20 @@ impl StorageOverrides { } } -impl Into>> for StorageOverrides { - fn into(self) -> Rc> { - self.to_rc_ptr() +impl From> for Rc>> { + fn from(storage_overrides: StorageOverrides) -> Rc>> { + storage_overrides.to_rc_ptr() } } impl ReadStorage for StorageOverrides { fn read_value(&mut self, key: &StorageKey) -> StorageValue { - // If the account is overrided, return the overrided value if any or zero. + // If the account is overridden, return the overridden value if any or zero. // Otherwise, return the value from the underlying storage. - self.overrided_account_state - .get(key.account()) - .and_then(|state| match state.get(key.key()) { - Some(v) => Some(v.clone()), - None => Some(H256::zero()), - }) - .unwrap_or_else(|| self.storage_view.read_value(key)) + self.overrided_account_state.get(key.account()).map_or_else( + || self.storage_view.read_value(key), + |state| state.get(key.key()).copied().unwrap_or_else(H256::zero), + ) } /// Only keys contained in the underlying storage will return `false`. If a key was diff --git a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs index f968f8d5af0f..72de76073d42 100644 --- a/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs +++ b/core/lib/zksync_core/src/api_server/execution_sandbox/apply.rs @@ -343,7 +343,7 @@ pub(super) fn apply_vm_in_sandbox( // Apply state override if let Some(state_override) = state_override { let mut storage_override = storage_view.as_ref().borrow_mut(); - storage_override.apply_state_override(state_override); + storage_override.apply_state_override(&state_override); } let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); From 1a39ddd68be36dafc82b3d9fe38ce23c5b1dff3e Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 14 Mar 2024 14:41:05 -0300 Subject: [PATCH 11/26] Cargo lock --- Cargo.lock | 1 + contracts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 048d0cd7f9cf..768ec293102d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8234,6 +8234,7 @@ dependencies = [ "prost", "rand 0.8.5", "reqwest", + "rlp", "serde", "serde_json", "serde_yaml", diff --git a/contracts b/contracts index e0a33ce73c4d..d85a73a1eeb5 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit e0a33ce73c4decd381446a6eb812b14c2ff69c47 +Subproject commit d85a73a1eeb5557343b7b44c6543aaf391d8b984 From 5c56d18daa65d6e2e1b31bc1ee1176990d6039e0 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 6 Jun 2024 12:08:38 -0300 Subject: [PATCH 12/26] zk fmt --- core/lib/types/src/api/state_override.rs | 6 +++--- core/node/api_server/src/execution_sandbox/apply.rs | 5 +++-- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/core/lib/types/src/api/state_override.rs b/core/lib/types/src/api/state_override.rs index 5faaae280d62..4ea60d7b0459 100644 --- a/core/lib/types/src/api/state_override.rs +++ b/core/lib/types/src/api/state_override.rs @@ -1,9 +1,9 @@ use std::{collections::HashMap, ops::Deref}; -use crate::Address; use serde::{Deserialize, Deserializer, Serialize}; -use zksync_basic_types::web3::Bytes; -use zksync_basic_types::{H256, U256}; +use zksync_basic_types::{web3::Bytes, H256, U256}; + +use crate::Address; /// Collection of overridden accounts, useful for `eth_estimateGas`. #[derive(Debug, Clone, Serialize, Deserialize)] diff --git a/core/node/api_server/src/execution_sandbox/apply.rs b/core/node/api_server/src/execution_sandbox/apply.rs index 18457c932510..4abcc8867505 100644 --- a/core/node/api_server/src/execution_sandbox/apply.rs +++ b/core/node/api_server/src/execution_sandbox/apply.rs @@ -21,8 +21,9 @@ use multivm::{ }; use tokio::runtime::Handle; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal, DalError}; -use zksync_state::StorageOverrides; -use zksync_state::{PostgresStorage, ReadStorage, StoragePtr, StorageView, WriteStorage}; +use zksync_state::{ + PostgresStorage, ReadStorage, StorageOverrides, StoragePtr, StorageView, WriteStorage, +}; use zksync_system_constants::{ SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_CURRENT_L2_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, From 945946ffd9cee3cce2ad86b7b0b2746a812104c2 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 6 Jun 2024 15:38:27 -0300 Subject: [PATCH 13/26] Revert submodule update to match main --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index e0a33ce73c4d..f3630fcb01ad 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit e0a33ce73c4decd381446a6eb812b14c2ff69c47 +Subproject commit f3630fcb01ad8b6e2e423a6f313abefe8502c3a2 From da57e99af0f50d4c25bf90877aa54bd5b432ccff Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 6 Jun 2024 16:32:55 -0300 Subject: [PATCH 14/26] contracts --- contracts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contracts b/contracts index f3630fcb01ad..8a70bbbc4812 160000 --- a/contracts +++ b/contracts @@ -1 +1 @@ -Subproject commit f3630fcb01ad8b6e2e423a6f313abefe8502c3a2 +Subproject commit 8a70bbbc48125f5bde6189b4e3c6a3ee79631678 From fd8af63b0df0c7c497063aff64228198195bcdfb Mon Sep 17 00:00:00 2001 From: Jrigada Date: Mon, 10 Jun 2024 10:06:11 -0300 Subject: [PATCH 15/26] fix CI --- core/node/api_server/src/execution_sandbox/tests.rs | 1 + core/node/api_server/src/web3/tests/vm.rs | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/core/node/api_server/src/execution_sandbox/tests.rs b/core/node/api_server/src/execution_sandbox/tests.rs index 9abe97f90254..1cf805bf11a6 100644 --- a/core/node/api_server/src/execution_sandbox/tests.rs +++ b/core/node/api_server/src/execution_sandbox/tests.rs @@ -195,6 +195,7 @@ async fn test_instantiating_vm(pool: ConnectionPool, block_args: BlockArgs &pool, transaction.clone(), block_args, + None, |_, received_tx, _| { assert_eq!(received_tx, transaction); }, diff --git a/core/node/api_server/src/web3/tests/vm.rs b/core/node/api_server/src/web3/tests/vm.rs index d0fa8f55f096..e1325ad8da3c 100644 --- a/core/node/api_server/src/web3/tests/vm.rs +++ b/core/node/api_server/src/web3/tests/vm.rs @@ -2,6 +2,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; +use api::state_override::{OverrideAccount, StateOverride}; use itertools::Itertools; use multivm::{ interface::{ExecutionResult, VmRevertReason}, @@ -674,7 +675,7 @@ impl HttpTest for EstimateGasWithStateOverrideTest { let pending_block_number = if self.snapshot_recovery { StorageInitialization::SNAPSHOT_RECOVERY_BLOCK + 2 } else { - MiniblockNumber(1) + L2BlockNumber(1) }; let gas_limit_threshold = self.gas_limit_threshold.clone(); tx_executor.set_call_responses(move |tx, block_args| { @@ -694,7 +695,7 @@ impl HttpTest for EstimateGasWithStateOverrideTest { tx_executor } - async fn test(&self, client: &HttpClient, _pool: &ConnectionPool) -> anyhow::Result<()> { + async fn test(&self, client: &HttpClient, _pool: &ConnectionPool) -> anyhow::Result<()> { // Transaction with balance override let l2_transaction = create_l2_transaction(10, 100); let mut call_request = CallRequest::from(l2_transaction); From 7182490d35411647aa4f21ab131bc13c9c1cc941 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Mon, 10 Jun 2024 10:27:46 -0300 Subject: [PATCH 16/26] change test function signature --- core/node/api_server/src/web3/tests/vm.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/core/node/api_server/src/web3/tests/vm.rs b/core/node/api_server/src/web3/tests/vm.rs index e1325ad8da3c..f1004dc55c3d 100644 --- a/core/node/api_server/src/web3/tests/vm.rs +++ b/core/node/api_server/src/web3/tests/vm.rs @@ -695,7 +695,11 @@ impl HttpTest for EstimateGasWithStateOverrideTest { tx_executor } - async fn test(&self, client: &HttpClient, _pool: &ConnectionPool) -> anyhow::Result<()> { + async fn test( + &self, + client: &DynClient, + _pool: &ConnectionPool, + ) -> anyhow::Result<()> { // Transaction with balance override let l2_transaction = create_l2_transaction(10, 100); let mut call_request = CallRequest::from(l2_transaction); From 4ac87e0fc1b46ae746016b6ed1ea8d985f4e6098 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Wed, 19 Jun 2024 09:49:44 -0300 Subject: [PATCH 17/26] Move StorageOverrides inside StorageView --- core/lib/state/src/storage_overrides.rs | 128 ++++++------------ core/lib/state/src/storage_view.rs | 64 +++++++-- .../api_server/src/execution_sandbox/apply.rs | 30 ++-- 3 files changed, 105 insertions(+), 117 deletions(-) diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 1c9fc8392cd0..7d69708d216f 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -1,72 +1,31 @@ use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; -use zksync_types::{ - api::state_override::{OverrideState, StateOverride}, - get_code_key, get_nonce_key, - utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, - AccountTreeId, StorageKey, StorageValue, H256, -}; -use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; +use zksync_types::{AccountTreeId, StorageKey, StorageValue, H256, U256}; +use zksync_utils::u256_to_h256; -use crate::{ReadStorage, StorageView, StorageViewMetrics, WriteStorage}; +use crate::ReadStorage; /// A storage view that allows to override some of the storage values. #[derive(Debug)] pub struct StorageOverrides { - storage_view: StorageView, + storage_handle: S, overrided_factory_deps: HashMap>, overrided_account_state: HashMap>, -} - -impl StorageOverrides { - /// Applies the state override to the storage. - pub fn apply_state_override(&mut self, state_override: &StateOverride) { - for (account, overrides) in state_override.iter() { - if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(account); - self.set_value(balance_key, u256_to_h256(balance)); - } - - if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(account); - - let full_nonce = self.read_value(&nonce_key); - let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); - let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - - self.set_value(nonce_key, u256_to_h256(new_full_nonce)); - } - - if let Some(code) = &overrides.code { - let code_key = get_code_key(account); - let code_hash = hash_bytecode(&code.0); - - self.set_value(code_key, code_hash); - self.store_factory_dep(code_hash, code.0.clone()); - } - - match &overrides.state { - Some(OverrideState::State(state)) => { - self.override_account_state(AccountTreeId::new(*account), state.clone()); - } - Some(OverrideState::StateDiff(state_diff)) => { - for (key, value) in state_diff { - self.set_value(StorageKey::new(AccountTreeId::new(*account), *key), *value); - } - } - None => {} - } - } - } + overrided_balance: HashMap, + overrided_nonce: HashMap, + overrided_code: HashMap, } impl StorageOverrides { /// Creates a new storage view based on the underlying storage. - pub fn new(storage_view: StorageView) -> Self { + pub fn new(storage: S) -> Self { Self { - storage_view, + storage_handle: storage, overrided_factory_deps: HashMap::new(), overrided_account_state: HashMap::new(), + overrided_balance: HashMap::new(), + overrided_nonce: HashMap::new(), + overrided_code: HashMap::new(), } } @@ -80,9 +39,19 @@ impl StorageOverrides { self.overrided_account_state.insert(account, state); } - /// Returns the current metrics. - pub fn metrics(&self) -> StorageViewMetrics { - self.storage_view.metrics() + /// Overrides an account balance. + pub fn store_balance_override(&mut self, key: StorageKey, balance: U256) { + self.overrided_balance.insert(key, balance); + } + + /// Overrides an account nonce. + pub fn store_nonce_override(&mut self, key: StorageKey, nonce: U256) { + self.overrided_nonce.insert(key, nonce); + } + + /// Overrides an account code. + pub fn store_code_overrrided(&mut self, key: StorageKey, code: H256) { + self.overrided_code.insert(key, code); } /// Make a Rc RefCell ptr to the storage @@ -91,54 +60,37 @@ impl StorageOverrides { } } -impl From> for Rc>> { - fn from(storage_overrides: StorageOverrides) -> Rc>> { - storage_overrides.to_rc_ptr() - } -} - impl ReadStorage for StorageOverrides { fn read_value(&mut self, key: &StorageKey) -> StorageValue { - // If the account is overridden, return the overridden value if any or zero. - // Otherwise, return the value from the underlying storage. + if let Some(balance) = self.overrided_balance.get(key) { + return u256_to_h256(*balance); + } + if let Some(code) = self.overrided_code.get(key) { + return *code; + } + + if let Some(nonce) = self.overrided_nonce.get(key) { + return u256_to_h256(*nonce); + } + self.overrided_account_state.get(key.account()).map_or_else( - || self.storage_view.read_value(key), + || self.storage_handle.read_value(key), |state| state.get(key.key()).copied().unwrap_or_else(H256::zero), ) } - /// Only keys contained in the underlying storage will return `false`. If a key was - /// inserted using [`Self::set_value()`], it will still return `true`. fn is_write_initial(&mut self, key: &StorageKey) -> bool { - self.storage_view.is_write_initial(key) + self.storage_handle.is_write_initial(key) } fn load_factory_dep(&mut self, hash: H256) -> Option> { self.overrided_factory_deps .get(&hash) .cloned() - .or_else(|| self.storage_view.load_factory_dep(hash)) + .or_else(|| self.storage_handle.load_factory_dep(hash)) } fn get_enumeration_index(&mut self, key: &StorageKey) -> Option { - self.storage_view.get_enumeration_index(key) - } -} - -impl WriteStorage for StorageOverrides { - fn set_value(&mut self, key: StorageKey, value: StorageValue) -> StorageValue { - self.storage_view.set_value(key, value) - } - - fn modified_storage_keys(&self) -> &HashMap { - self.storage_view.modified_storage_keys() - } - - fn missed_storage_invocations(&self) -> usize { - self.storage_view.missed_storage_invocations() - } - - fn read_storage_keys(&self) -> &HashMap { - self.storage_view.read_storage_keys() + self.storage_handle.get_enumeration_index(key) } } diff --git a/core/lib/state/src/storage_view.rs b/core/lib/state/src/storage_view.rs index 7756f6007eec..e71b43d008fb 100644 --- a/core/lib/state/src/storage_view.rs +++ b/core/lib/state/src/storage_view.rs @@ -6,9 +6,15 @@ use std::{ time::{Duration, Instant}, }; -use zksync_types::{StorageKey, StorageValue, H256}; +use zksync_types::{ + api::state_override::{OverrideState, StateOverride}, + get_code_key, get_nonce_key, + utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, + AccountTreeId, StorageKey, StorageValue, H256, +}; +use zksync_utils::{bytecode::hash_bytecode, h256_to_u256}; -use crate::{ReadStorage, WriteStorage}; +use crate::{ReadStorage, StorageOverrides, WriteStorage}; /// Metrics for [`StorageView`]. #[derive(Debug, Default, Clone, Copy)] @@ -42,7 +48,6 @@ pub struct StorageViewMetrics { /// the only shared part is the read storage keys cache. #[derive(Debug)] pub struct StorageView { - storage_handle: S, // Used for caching and to get the list/count of modified keys modified_storage_keys: HashMap, // Used purely for caching @@ -50,6 +55,7 @@ pub struct StorageView { // Cache for `contains_key()` checks. The cache is only valid within one L1 batch execution. initial_writes_cache: HashMap, metrics: StorageViewMetrics, + storage_overrides: StorageOverrides, } impl StorageView { @@ -85,14 +91,54 @@ where } impl StorageView { + /// Applies the state override to the storageOverride. + pub fn apply_state_override(&mut self, state_override: &StateOverride) { + for (account, overrides) in state_override.iter() { + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(account); + self.storage_overrides + .store_balance_override(balance_key, balance); + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(account); + + let full_nonce = self.read_value(&nonce_key); + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + self.storage_overrides + .store_nonce_override(nonce_key, new_full_nonce); + } + + if let Some(code) = &overrides.code { + let code_key = get_code_key(account); + let code_hash = hash_bytecode(&code.0); + self.storage_overrides + .store_code_overrrided(code_key, code_hash); + self.storage_overrides + .store_factory_dep(code_hash, code.0.clone()); + } + + match &overrides.state { + Some(OverrideState::State(state)) => { + self.storage_overrides + .override_account_state(AccountTreeId::new(*account), state.clone()); + } + Some(OverrideState::StateDiff(_state_diff)) => { + // TODO: Implement state diff override + } + None => {} + } + } + } /// Creates a new storage view based on the underlying storage. pub fn new(storage_handle: S) -> Self { Self { - storage_handle, modified_storage_keys: HashMap::new(), read_storage_keys: HashMap::new(), initial_writes_cache: HashMap::new(), metrics: StorageViewMetrics::default(), + storage_overrides: StorageOverrides::new(storage_handle), } } @@ -104,7 +150,7 @@ impl StorageView { .get(key) .or_else(|| self.read_storage_keys.get(key)); cached_value.copied().unwrap_or_else(|| { - let value = self.storage_handle.read_value(key); + let value = self.storage_overrides.read_value(key); self.read_storage_keys.insert(*key, value); self.metrics.time_spent_on_storage_missed += started_at.elapsed(); self.metrics.storage_invocations_missed += 1; @@ -156,18 +202,18 @@ impl ReadStorage for StorageView { if let Some(&is_write_initial) = self.initial_writes_cache.get(key) { is_write_initial } else { - let is_write_initial = self.storage_handle.is_write_initial(key); + let is_write_initial = self.storage_overrides.is_write_initial(key); self.initial_writes_cache.insert(*key, is_write_initial); is_write_initial } } fn load_factory_dep(&mut self, hash: H256) -> Option> { - self.storage_handle.load_factory_dep(hash) + self.storage_overrides.load_factory_dep(hash) } fn get_enumeration_index(&mut self, key: &StorageKey) -> Option { - self.storage_handle.get_enumeration_index(key) + self.storage_overrides.get_enumeration_index(key) } } @@ -181,7 +227,7 @@ impl WriteStorage for StorageView { self.metrics.set_value_storage_invocations += 1; let original = self.get_value_no_log(&key); - tracing::trace!( + tracing::info!( "write value {:?} value: {:?} original value: {:?} ({:?}/{:?})", key.hashed_key().0, value, diff --git a/core/node/api_server/src/execution_sandbox/apply.rs b/core/node/api_server/src/execution_sandbox/apply.rs index 4abcc8867505..15c5a3dda2d9 100644 --- a/core/node/api_server/src/execution_sandbox/apply.rs +++ b/core/node/api_server/src/execution_sandbox/apply.rs @@ -6,11 +6,7 @@ //! //! This module is intended to be blocking. -use std::{ - cell::RefCell, - rc::Rc, - time::{Duration, Instant}, -}; +use std::time::{Duration, Instant}; use anyhow::Context as _; use multivm::{ @@ -21,9 +17,7 @@ use multivm::{ }; use tokio::runtime::Handle; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal, DalError}; -use zksync_state::{ - PostgresStorage, ReadStorage, StorageOverrides, StoragePtr, StorageView, WriteStorage, -}; +use zksync_state::{PostgresStorage, ReadStorage, StoragePtr, StorageView, WriteStorage}; use zksync_system_constants::{ SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_CURRENT_L2_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, @@ -44,7 +38,7 @@ use super::{ BlockArgs, TxExecutionArgs, TxSharedArgs, VmPermit, }; -type BoxedVm<'a> = Box>, HistoryDisabled>>; +type BoxedVm<'a> = Box>, HistoryDisabled>>; #[derive(Debug)] struct Sandbox<'a> { @@ -265,10 +259,7 @@ impl<'a> Sandbox<'a> { mut self, tx: &Transaction, adjust_pubdata_price: bool, - ) -> ( - BoxedVm<'a>, - StoragePtr>>, - ) { + ) -> (BoxedVm<'a>, StoragePtr>>) { self.setup_storage_view(tx); let protocol_version = self.system_env.version; if adjust_pubdata_price { @@ -280,16 +271,15 @@ impl<'a> Sandbox<'a> { ); }; - let storage_overrides: Rc>>> = - StorageOverrides::new(self.storage_view).into(); + let storage_view = self.storage_view.to_rc_ptr(); let vm = Box::new(VmInstance::new_with_specific_version( self.l1_batch_env, self.system_env, - storage_overrides.clone(), + storage_view.clone(), protocol_version.into_api_vm_version(), )); - (vm, storage_overrides) + (vm, storage_view) } } @@ -307,7 +297,7 @@ pub(super) fn apply_vm_in_sandbox( block_args: BlockArgs, state_override: Option, apply: impl FnOnce( - &mut VmInstance>, HistoryDisabled>, + &mut VmInstance>, HistoryDisabled>, Transaction, ProtocolVersionId, ) -> T, @@ -345,8 +335,8 @@ pub(super) fn apply_vm_in_sandbox( // Apply state override if let Some(state_override) = state_override { - let mut storage_override = storage_view.as_ref().borrow_mut(); - storage_override.apply_state_override(&state_override); + let mut storage_view = storage_view.as_ref().borrow_mut(); + storage_view.apply_state_override(&state_override); } let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); From f4c0189300af24eea83481a215450dcd3a5630ea Mon Sep 17 00:00:00 2001 From: Jrigada Date: Wed, 19 Jun 2024 16:18:29 -0300 Subject: [PATCH 18/26] Apply earlier the storageOverrides --- core/lib/state/src/lib.rs | 7 ++ core/lib/state/src/storage_overrides.rs | 71 ++++++++++++++----- core/lib/state/src/storage_view.rs | 69 ++++-------------- core/lib/vm_utils/src/lib.rs | 11 +-- .../api_server/src/execution_sandbox/apply.rs | 41 +++++++---- .../ts-integration/tests/api/web3.test.ts | 4 +- 6 files changed, 108 insertions(+), 95 deletions(-) diff --git a/core/lib/state/src/lib.rs b/core/lib/state/src/lib.rs index 12ddbe83f64f..afb192290c18 100644 --- a/core/lib/state/src/lib.rs +++ b/core/lib/state/src/lib.rs @@ -12,6 +12,7 @@ use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; use zksync_types::{ + api::state_override::StateOverride, get_known_code_key, storage::{StorageKey, StorageValue}, H256, @@ -89,3 +90,9 @@ pub trait WriteStorage: ReadStorage { /// Smart pointer to [`WriteStorage`]. pub type StoragePtr = Rc>; + +/// Functionality to override the storage state. +pub trait OverrideStorage { + /// Apply state override to the storage. + fn apply_state_override(&mut self, overrides: &StateOverride); +} diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 7d69708d216f..5c1362e095fc 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -1,9 +1,14 @@ use std::{cell::RefCell, collections::HashMap, fmt, rc::Rc}; -use zksync_types::{AccountTreeId, StorageKey, StorageValue, H256, U256}; -use zksync_utils::u256_to_h256; +use zksync_types::{ + api::state_override::{OverrideState, StateOverride}, + get_code_key, get_nonce_key, + utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, + AccountTreeId, StorageKey, StorageValue, H256, U256, +}; +use zksync_utils::{bytecode::hash_bytecode, h256_to_u256, u256_to_h256}; -use crate::ReadStorage; +use crate::{OverrideStorage, ReadStorage}; /// A storage view that allows to override some of the storage values. #[derive(Debug)] @@ -39,21 +44,6 @@ impl StorageOverrides { self.overrided_account_state.insert(account, state); } - /// Overrides an account balance. - pub fn store_balance_override(&mut self, key: StorageKey, balance: U256) { - self.overrided_balance.insert(key, balance); - } - - /// Overrides an account nonce. - pub fn store_nonce_override(&mut self, key: StorageKey, nonce: U256) { - self.overrided_nonce.insert(key, nonce); - } - - /// Overrides an account code. - pub fn store_code_overrrided(&mut self, key: StorageKey, code: H256) { - self.overrided_code.insert(key, code); - } - /// Make a Rc RefCell ptr to the storage pub fn to_rc_ptr(self) -> Rc> { Rc::new(RefCell::new(self)) @@ -72,7 +62,8 @@ impl ReadStorage for StorageOverrides { if let Some(nonce) = self.overrided_nonce.get(key) { return u256_to_h256(*nonce); } - + // If the account is overridden, return the overridden value if any or zero. + // Otherwise, return the value from the underlying storage. self.overrided_account_state.get(key.account()).map_or_else( || self.storage_handle.read_value(key), |state| state.get(key.key()).copied().unwrap_or_else(H256::zero), @@ -94,3 +85,45 @@ impl ReadStorage for StorageOverrides { self.storage_handle.get_enumeration_index(key) } } + +impl OverrideStorage for StorageOverrides { + fn apply_state_override(&mut self, state_override: &StateOverride) { + for (account, overrides) in state_override.iter() { + if let Some(balance) = overrides.balance { + let balance_key = storage_key_for_eth_balance(account); + self.overrided_balance.insert(balance_key, balance); + } + + if let Some(nonce) = overrides.nonce { + let nonce_key = get_nonce_key(account); + let full_nonce = self.read_value(&nonce_key); + let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); + let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); + self.overrided_nonce.insert(nonce_key, new_full_nonce); + } + + if let Some(code) = &overrides.code { + let code_key = get_code_key(account); + let code_hash = hash_bytecode(&code.0); + self.overrided_code.insert(code_key, code_hash); + self.store_factory_dep(code_hash, code.0.clone()); + } + + match &overrides.state { + Some(OverrideState::State(state)) => { + self.override_account_state(AccountTreeId::new(*account), state.clone()); + } + Some(OverrideState::StateDiff(state_diff)) => { + for (key, value) in state_diff { + let account_state = self + .overrided_account_state + .entry(AccountTreeId::new(*account)) + .or_default(); + account_state.insert(*key, *value); + } + } + None => {} + } + } + } +} diff --git a/core/lib/state/src/storage_view.rs b/core/lib/state/src/storage_view.rs index e71b43d008fb..16a70ea1966a 100644 --- a/core/lib/state/src/storage_view.rs +++ b/core/lib/state/src/storage_view.rs @@ -6,15 +6,9 @@ use std::{ time::{Duration, Instant}, }; -use zksync_types::{ - api::state_override::{OverrideState, StateOverride}, - get_code_key, get_nonce_key, - utils::{decompose_full_nonce, nonces_to_full_nonce, storage_key_for_eth_balance}, - AccountTreeId, StorageKey, StorageValue, H256, -}; -use zksync_utils::{bytecode::hash_bytecode, h256_to_u256}; +use zksync_types::{StorageKey, StorageValue, H256}; -use crate::{ReadStorage, StorageOverrides, WriteStorage}; +use crate::{ReadStorage, WriteStorage}; /// Metrics for [`StorageView`]. #[derive(Debug, Default, Clone, Copy)] @@ -48,6 +42,7 @@ pub struct StorageViewMetrics { /// the only shared part is the read storage keys cache. #[derive(Debug)] pub struct StorageView { + storage_handle: S, // Used for caching and to get the list/count of modified keys modified_storage_keys: HashMap, // Used purely for caching @@ -55,7 +50,6 @@ pub struct StorageView { // Cache for `contains_key()` checks. The cache is only valid within one L1 batch execution. initial_writes_cache: HashMap, metrics: StorageViewMetrics, - storage_overrides: StorageOverrides, } impl StorageView { @@ -63,6 +57,11 @@ impl StorageView { pub fn modified_storage_keys(&self) -> &HashMap { &self.modified_storage_keys } + + /// Returns the underlying storage handle + pub fn storage_handle_mut(&mut self) -> &mut S { + &mut self.storage_handle + } } impl ReadStorage for Box @@ -91,54 +90,14 @@ where } impl StorageView { - /// Applies the state override to the storageOverride. - pub fn apply_state_override(&mut self, state_override: &StateOverride) { - for (account, overrides) in state_override.iter() { - if let Some(balance) = overrides.balance { - let balance_key = storage_key_for_eth_balance(account); - self.storage_overrides - .store_balance_override(balance_key, balance); - } - - if let Some(nonce) = overrides.nonce { - let nonce_key = get_nonce_key(account); - - let full_nonce = self.read_value(&nonce_key); - let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); - let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - self.storage_overrides - .store_nonce_override(nonce_key, new_full_nonce); - } - - if let Some(code) = &overrides.code { - let code_key = get_code_key(account); - let code_hash = hash_bytecode(&code.0); - self.storage_overrides - .store_code_overrrided(code_key, code_hash); - self.storage_overrides - .store_factory_dep(code_hash, code.0.clone()); - } - - match &overrides.state { - Some(OverrideState::State(state)) => { - self.storage_overrides - .override_account_state(AccountTreeId::new(*account), state.clone()); - } - Some(OverrideState::StateDiff(_state_diff)) => { - // TODO: Implement state diff override - } - None => {} - } - } - } /// Creates a new storage view based on the underlying storage. pub fn new(storage_handle: S) -> Self { Self { + storage_handle, modified_storage_keys: HashMap::new(), read_storage_keys: HashMap::new(), initial_writes_cache: HashMap::new(), metrics: StorageViewMetrics::default(), - storage_overrides: StorageOverrides::new(storage_handle), } } @@ -150,7 +109,7 @@ impl StorageView { .get(key) .or_else(|| self.read_storage_keys.get(key)); cached_value.copied().unwrap_or_else(|| { - let value = self.storage_overrides.read_value(key); + let value = self.storage_handle.read_value(key); self.read_storage_keys.insert(*key, value); self.metrics.time_spent_on_storage_missed += started_at.elapsed(); self.metrics.storage_invocations_missed += 1; @@ -202,18 +161,18 @@ impl ReadStorage for StorageView { if let Some(&is_write_initial) = self.initial_writes_cache.get(key) { is_write_initial } else { - let is_write_initial = self.storage_overrides.is_write_initial(key); + let is_write_initial = self.storage_handle.is_write_initial(key); self.initial_writes_cache.insert(*key, is_write_initial); is_write_initial } } fn load_factory_dep(&mut self, hash: H256) -> Option> { - self.storage_overrides.load_factory_dep(hash) + self.storage_handle.load_factory_dep(hash) } fn get_enumeration_index(&mut self, key: &StorageKey) -> Option { - self.storage_overrides.get_enumeration_index(key) + self.storage_handle.get_enumeration_index(key) } } @@ -227,7 +186,7 @@ impl WriteStorage for StorageView { self.metrics.set_value_storage_invocations += 1; let original = self.get_value_no_log(&key); - tracing::info!( + tracing::trace!( "write value {:?} value: {:?} original value: {:?} ({:?}/{:?})", key.hashed_key().0, value, diff --git a/core/lib/vm_utils/src/lib.rs b/core/lib/vm_utils/src/lib.rs index 2dcf186d48d3..d5e409853c3f 100644 --- a/core/lib/vm_utils/src/lib.rs +++ b/core/lib/vm_utils/src/lib.rs @@ -8,14 +8,14 @@ use multivm::{ }; use tokio::runtime::Handle; use zksync_dal::{Connection, Core}; -use zksync_state::{PostgresStorage, StoragePtr, StorageView, WriteStorage}; +use zksync_state::{PostgresStorage, StorageOverrides, StoragePtr, StorageView, WriteStorage}; use zksync_types::{L1BatchNumber, L2ChainId, Transaction}; use crate::storage::L1BatchParamsProvider; pub type VmAndStorage<'a> = ( - VmInstance>, HistoryEnabled>, - StoragePtr>>, + VmInstance>>, HistoryEnabled>, + StoragePtr>>>, ); pub fn create_vm( @@ -23,7 +23,7 @@ pub fn create_vm( l1_batch_number: L1BatchNumber, mut connection: Connection<'_, Core>, l2_chain_id: L2ChainId, -) -> anyhow::Result { +) -> anyhow::Result> { let l1_batch_params_provider = rt_handle .block_on(L1BatchParamsProvider::new(&mut connection)) .context("failed initializing L1 batch params provider")?; @@ -51,7 +51,8 @@ pub fn create_vm( let storage_l2_block_number = first_l2_block_in_batch.number() - 1; let pg_storage = PostgresStorage::new(rt_handle.clone(), connection, storage_l2_block_number, true); - let storage_view = StorageView::new(pg_storage).to_rc_ptr(); + let storage_overrides = StorageOverrides::new(pg_storage); + let storage_view = StorageView::new(storage_overrides).to_rc_ptr(); let vm = VmInstance::new(l1_batch_env, system_env, storage_view.clone()); Ok((vm, storage_view)) diff --git a/core/node/api_server/src/execution_sandbox/apply.rs b/core/node/api_server/src/execution_sandbox/apply.rs index 15c5a3dda2d9..f6aab1a4b147 100644 --- a/core/node/api_server/src/execution_sandbox/apply.rs +++ b/core/node/api_server/src/execution_sandbox/apply.rs @@ -17,7 +17,10 @@ use multivm::{ }; use tokio::runtime::Handle; use zksync_dal::{Connection, ConnectionPool, Core, CoreDal, DalError}; -use zksync_state::{PostgresStorage, ReadStorage, StoragePtr, StorageView, WriteStorage}; +use zksync_state::{ + OverrideStorage, PostgresStorage, ReadStorage, StorageOverrides, StoragePtr, StorageView, + WriteStorage, +}; use zksync_system_constants::{ SYSTEM_CONTEXT_ADDRESS, SYSTEM_CONTEXT_CURRENT_L2_BLOCK_INFO_POSITION, SYSTEM_CONTEXT_CURRENT_TX_ROLLING_HASH_POSITION, ZKPORTER_IS_AVAILABLE, @@ -38,7 +41,8 @@ use super::{ BlockArgs, TxExecutionArgs, TxSharedArgs, VmPermit, }; -type BoxedVm<'a> = Box>, HistoryDisabled>>; +type BoxedVm<'a> = + Box>>, HistoryDisabled>>; #[derive(Debug)] struct Sandbox<'a> { @@ -46,7 +50,7 @@ struct Sandbox<'a> { l1_batch_env: L1BatchEnv, execution_args: &'a TxExecutionArgs, l2_block_info_to_reset: Option, - storage_view: StorageView>, + storage_view: StorageView>>, } impl<'a> Sandbox<'a> { @@ -90,7 +94,9 @@ impl<'a> Sandbox<'a> { .context("cannot create `PostgresStorage`")? .with_caches(shared_args.caches.clone()); - let storage_view = StorageView::new(storage); + let storage_overrides = StorageOverrides::new(storage); + + let storage_view = StorageView::new(storage_overrides); let (system_env, l1_batch_env) = Self::prepare_env( shared_args, execution_args, @@ -259,7 +265,20 @@ impl<'a> Sandbox<'a> { mut self, tx: &Transaction, adjust_pubdata_price: bool, - ) -> (BoxedVm<'a>, StoragePtr>>) { + state_override: Option, + ) -> ( + BoxedVm<'a>, + StoragePtr>>>, + ) { + // Apply state override + if let Some(state_override) = state_override { + // storage_view_borrow.disable_cache(); + let storage_overrides: &mut StorageOverrides = + self.storage_view.storage_handle_mut(); + + // Apply the state override using the borrowed storage_view + storage_overrides.apply_state_override(&state_override); + } self.setup_storage_view(tx); let protocol_version = self.system_env.version; if adjust_pubdata_price { @@ -294,10 +313,10 @@ pub(super) fn apply_vm_in_sandbox( execution_args: &TxExecutionArgs, connection_pool: &ConnectionPool, tx: Transaction, - block_args: BlockArgs, + block_args: BlockArgs, // Block arguments for the transaction. state_override: Option, apply: impl FnOnce( - &mut VmInstance>, HistoryDisabled>, + &mut VmInstance>>, HistoryDisabled>, Transaction, ProtocolVersionId, ) -> T, @@ -322,7 +341,7 @@ pub(super) fn apply_vm_in_sandbox( block_args, ))?; let protocol_version = sandbox.system_env.version; - let (mut vm, storage_view) = sandbox.into_vm(&tx, adjust_pubdata_price); + let (mut vm, storage_view) = sandbox.into_vm(&tx, adjust_pubdata_price, state_override); SANDBOX_METRICS.sandbox[&SandboxStage::Initialization].observe(stage_started_at.elapsed()); span.exit(); @@ -333,12 +352,6 @@ pub(super) fn apply_vm_in_sandbox( tx.nonce().unwrap_or(Nonce(0)) ); - // Apply state override - if let Some(state_override) = state_override { - let mut storage_view = storage_view.as_ref().borrow_mut(); - storage_view.apply_state_override(&state_override); - } - let execution_latency = SANDBOX_METRICS.sandbox[&SandboxStage::Execution].start(); let result = apply(&mut vm, tx, protocol_version); let vm_execution_took = execution_latency.observe(); diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index 3da754a9b371..e8dc0dc31996 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -956,10 +956,10 @@ describe('web3 API compatibility tests', () => { value: amount.toHexString() }, 'latest', - //override with maximum balance + //override with the balance needed to send the transaction { [alice.address]: { - balance: '0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff' + balance: amount.toHexString() } } ]); From f023a17c2f7a17eddd748cfbd67d6ecf3ac65ae3 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Wed, 19 Jun 2024 16:27:43 -0300 Subject: [PATCH 19/26] fix spelling --- core/node/api_server/src/execution_sandbox/apply.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/node/api_server/src/execution_sandbox/apply.rs b/core/node/api_server/src/execution_sandbox/apply.rs index f6aab1a4b147..cf669d7d4ef8 100644 --- a/core/node/api_server/src/execution_sandbox/apply.rs +++ b/core/node/api_server/src/execution_sandbox/apply.rs @@ -272,11 +272,10 @@ impl<'a> Sandbox<'a> { ) { // Apply state override if let Some(state_override) = state_override { - // storage_view_borrow.disable_cache(); let storage_overrides: &mut StorageOverrides = self.storage_view.storage_handle_mut(); - // Apply the state override using the borrowed storage_view + // Apply the state override storage_overrides.apply_state_override(&state_override); } self.setup_storage_view(tx); From 4fee27ad6ccaee990b32f12aea93affdf94fd38a Mon Sep 17 00:00:00 2001 From: Jrigada Date: Fri, 28 Jun 2024 13:12:00 -0300 Subject: [PATCH 20/26] storageview implement overrideStorage trait --- core/lib/state/src/storage_view.rs | 15 ++++++++------- .../api_server/src/execution_sandbox/apply.rs | 5 +---- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/core/lib/state/src/storage_view.rs b/core/lib/state/src/storage_view.rs index 16a70ea1966a..54b7b11d7cde 100644 --- a/core/lib/state/src/storage_view.rs +++ b/core/lib/state/src/storage_view.rs @@ -6,9 +6,9 @@ use std::{ time::{Duration, Instant}, }; -use zksync_types::{StorageKey, StorageValue, H256}; +use zksync_types::{api::state_override::StateOverride, StorageKey, StorageValue, H256}; -use crate::{ReadStorage, WriteStorage}; +use crate::{OverrideStorage, ReadStorage, WriteStorage}; /// Metrics for [`StorageView`]. #[derive(Debug, Default, Clone, Copy)] @@ -57,11 +57,6 @@ impl StorageView { pub fn modified_storage_keys(&self) -> &HashMap { &self.modified_storage_keys } - - /// Returns the underlying storage handle - pub fn storage_handle_mut(&mut self) -> &mut S { - &mut self.storage_handle - } } impl ReadStorage for Box @@ -209,6 +204,12 @@ impl WriteStorage for StorageView { } } +impl OverrideStorage for StorageView { + fn apply_state_override(&mut self, state_override: &StateOverride) { + self.storage_handle.apply_state_override(state_override); + } +} + #[cfg(test)] mod test { use zksync_types::{AccountTreeId, Address, H256}; diff --git a/core/node/api_server/src/execution_sandbox/apply.rs b/core/node/api_server/src/execution_sandbox/apply.rs index 8965c1fa913d..c30e5bc36c86 100644 --- a/core/node/api_server/src/execution_sandbox/apply.rs +++ b/core/node/api_server/src/execution_sandbox/apply.rs @@ -272,11 +272,8 @@ impl<'a> Sandbox<'a> { ) { // Apply state override if let Some(state_override) = state_override { - let storage_overrides: &mut StorageOverrides = - self.storage_view.storage_handle_mut(); - // Apply the state override - storage_overrides.apply_state_override(&state_override); + self.storage_view.apply_state_override(&state_override); } self.setup_storage_view(tx); let protocol_version = self.system_env.version; From 157577b62dd0402775108ea591d49a532e618350 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Mon, 1 Jul 2024 15:57:51 -0300 Subject: [PATCH 21/26] add logic for stateDiff and State with tests, and added state_override to eth_call and estimate_fee and estimate_l1_to_l2_gas --- core/lib/state/src/storage_overrides.rs | 72 ++++++---- core/lib/state/src/storage_view.rs | 7 - core/lib/vm_utils/src/lib.rs | 2 +- core/lib/web3_decl/src/namespaces/eth.rs | 7 +- core/lib/web3_decl/src/namespaces/zks.rs | 16 ++- .../src/execution_sandbox/execute.rs | 3 +- core/node/api_server/src/tx_sender/mod.rs | 2 + .../web3/backend_jsonrpsee/namespaces/eth.rs | 9 +- .../web3/backend_jsonrpsee/namespaces/zks.rs | 21 ++- .../api_server/src/web3/namespaces/debug.rs | 1 + .../api_server/src/web3/namespaces/eth.rs | 3 +- .../api_server/src/web3/namespaces/zks.rs | 28 +++- core/node/api_server/src/web3/tests/vm.rs | 17 ++- .../src/sdk/operations/deploy_contract.rs | 2 +- .../src/sdk/operations/execute_contract.rs | 2 +- .../loadnext/src/sdk/operations/transfer.rs | 2 +- core/tests/loadnext/src/sdk/wallet.rs | 2 +- .../state-override/StateOverrideTest.sol | 27 ++++ .../ts-integration/tests/api/web3.test.ts | 131 +++++++++++++++++- 19 files changed, 283 insertions(+), 71 deletions(-) create mode 100644 core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 5c1362e095fc..f6a45d6a6484 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -14,11 +14,12 @@ use crate::{OverrideStorage, ReadStorage}; #[derive(Debug)] pub struct StorageOverrides { storage_handle: S, - overrided_factory_deps: HashMap>, - overrided_account_state: HashMap>, - overrided_balance: HashMap, - overrided_nonce: HashMap, - overrided_code: HashMap, + overridden_factory_deps: HashMap>, + overridden_account_state: HashMap>, + overriden_account_state_diff: HashMap>, + overridden_balance: HashMap, + overridden_nonce: HashMap, + overridden_code: HashMap, } impl StorageOverrides { @@ -26,22 +27,33 @@ impl StorageOverrides { pub fn new(storage: S) -> Self { Self { storage_handle: storage, - overrided_factory_deps: HashMap::new(), - overrided_account_state: HashMap::new(), - overrided_balance: HashMap::new(), - overrided_nonce: HashMap::new(), - overrided_code: HashMap::new(), + overridden_factory_deps: HashMap::new(), + overridden_account_state: HashMap::new(), + overriden_account_state_diff: HashMap::new(), + overridden_balance: HashMap::new(), + overridden_nonce: HashMap::new(), + overridden_code: HashMap::new(), } } /// Overrides a factory dependency code. pub fn store_factory_dep(&mut self, hash: H256, code: Vec) { - self.overrided_factory_deps.insert(hash, code); + self.overridden_factory_deps.insert(hash, code); } /// Overrides an account entire state. pub fn override_account_state(&mut self, account: AccountTreeId, state: HashMap) { - self.overrided_account_state.insert(account, state); + self.overridden_account_state.insert(account, state); + } + + /// Overrides an account state diff. + pub fn override_account_state_diff( + &mut self, + account: AccountTreeId, + state_diff: HashMap, + ) { + self.overriden_account_state_diff + .insert(account, state_diff); } /// Make a Rc RefCell ptr to the storage @@ -52,22 +64,30 @@ impl StorageOverrides { impl ReadStorage for StorageOverrides { fn read_value(&mut self, key: &StorageKey) -> StorageValue { - if let Some(balance) = self.overrided_balance.get(key) { + if let Some(balance) = self.overridden_balance.get(key) { return u256_to_h256(*balance); } - if let Some(code) = self.overrided_code.get(key) { + if let Some(code) = self.overridden_code.get(key) { return *code; } - if let Some(nonce) = self.overrided_nonce.get(key) { + if let Some(nonce) = self.overridden_nonce.get(key) { return u256_to_h256(*nonce); } - // If the account is overridden, return the overridden value if any or zero. - // Otherwise, return the value from the underlying storage. - self.overrided_account_state.get(key.account()).map_or_else( - || self.storage_handle.read_value(key), - |state| state.get(key.key()).copied().unwrap_or_else(H256::zero), - ) + + if let Some(account_state) = self.overridden_account_state.get(key.account()) { + if let Some(value) = account_state.get(key.key()) { + return *value; + } + } + + if let Some(account_state_diff) = self.overriden_account_state_diff.get(key.account()) { + if let Some(value) = account_state_diff.get(key.key()) { + return *value; + } + } + + self.storage_handle.read_value(key) } fn is_write_initial(&mut self, key: &StorageKey) -> bool { @@ -75,7 +95,7 @@ impl ReadStorage for StorageOverrides { } fn load_factory_dep(&mut self, hash: H256) -> Option> { - self.overrided_factory_deps + self.overridden_factory_deps .get(&hash) .cloned() .or_else(|| self.storage_handle.load_factory_dep(hash)) @@ -91,7 +111,7 @@ impl OverrideStorage for StorageOverrides { for (account, overrides) in state_override.iter() { if let Some(balance) = overrides.balance { let balance_key = storage_key_for_eth_balance(account); - self.overrided_balance.insert(balance_key, balance); + self.overridden_balance.insert(balance_key, balance); } if let Some(nonce) = overrides.nonce { @@ -99,13 +119,13 @@ impl OverrideStorage for StorageOverrides { let full_nonce = self.read_value(&nonce_key); let (_, deployment_nonce) = decompose_full_nonce(h256_to_u256(full_nonce)); let new_full_nonce = nonces_to_full_nonce(nonce, deployment_nonce); - self.overrided_nonce.insert(nonce_key, new_full_nonce); + self.overridden_nonce.insert(nonce_key, new_full_nonce); } if let Some(code) = &overrides.code { let code_key = get_code_key(account); let code_hash = hash_bytecode(&code.0); - self.overrided_code.insert(code_key, code_hash); + self.overridden_code.insert(code_key, code_hash); self.store_factory_dep(code_hash, code.0.clone()); } @@ -116,7 +136,7 @@ impl OverrideStorage for StorageOverrides { Some(OverrideState::StateDiff(state_diff)) => { for (key, value) in state_diff { let account_state = self - .overrided_account_state + .overridden_account_state .entry(AccountTreeId::new(*account)) .or_default(); account_state.insert(*key, *value); diff --git a/core/lib/state/src/storage_view.rs b/core/lib/state/src/storage_view.rs index 54b7b11d7cde..1150d9cf8cf6 100644 --- a/core/lib/state/src/storage_view.rs +++ b/core/lib/state/src/storage_view.rs @@ -52,13 +52,6 @@ pub struct StorageView { metrics: StorageViewMetrics, } -impl StorageView { - /// Returns the modified storage keys - pub fn modified_storage_keys(&self) -> &HashMap { - &self.modified_storage_keys - } -} - impl ReadStorage for Box where S: ReadStorage + ?Sized, diff --git a/core/lib/vm_utils/src/lib.rs b/core/lib/vm_utils/src/lib.rs index 9aac67c91daa..374407b69551 100644 --- a/core/lib/vm_utils/src/lib.rs +++ b/core/lib/vm_utils/src/lib.rs @@ -23,7 +23,7 @@ pub fn create_vm( l1_batch_number: L1BatchNumber, mut connection: Connection<'_, Core>, l2_chain_id: L2ChainId, -) -> anyhow::Result> { +) -> anyhow::Result { let l1_batch_params_provider = rt_handle .block_on(L1BatchParamsProvider::new(&mut connection)) .context("failed initializing L1 batch params provider")?; diff --git a/core/lib/web3_decl/src/namespaces/eth.rs b/core/lib/web3_decl/src/namespaces/eth.rs index 3d3b8682d81d..10443443958b 100644 --- a/core/lib/web3_decl/src/namespaces/eth.rs +++ b/core/lib/web3_decl/src/namespaces/eth.rs @@ -34,7 +34,12 @@ pub trait EthNamespace { async fn chain_id(&self) -> RpcResult; #[method(name = "call")] - async fn call(&self, req: CallRequest, block: Option) -> RpcResult; + async fn call( + &self, + req: CallRequest, + block: Option, + state_override: Option, + ) -> RpcResult; #[method(name = "estimateGas")] async fn estimate_gas( diff --git a/core/lib/web3_decl/src/namespaces/zks.rs b/core/lib/web3_decl/src/namespaces/zks.rs index b6861a9d2dd7..6f443dbded6a 100644 --- a/core/lib/web3_decl/src/namespaces/zks.rs +++ b/core/lib/web3_decl/src/namespaces/zks.rs @@ -5,8 +5,8 @@ use jsonrpsee::core::RpcResult; use jsonrpsee::proc_macros::rpc; use zksync_types::{ api::{ - BlockDetails, BridgeAddresses, L1BatchDetails, L2ToL1LogProof, Proof, ProtocolVersion, - TransactionDetailedResult, TransactionDetails, + state_override::StateOverride, BlockDetails, BridgeAddresses, L1BatchDetails, + L2ToL1LogProof, Proof, ProtocolVersion, TransactionDetailedResult, TransactionDetails, }, fee::Fee, fee_model::{FeeParams, PubdataIndependentBatchFeeModelInput}, @@ -29,10 +29,18 @@ use crate::{ )] pub trait ZksNamespace { #[method(name = "estimateFee")] - async fn estimate_fee(&self, req: CallRequest) -> RpcResult; + async fn estimate_fee( + &self, + req: CallRequest, + state_override: Option, + ) -> RpcResult; #[method(name = "estimateGasL1ToL2")] - async fn estimate_gas_l1_to_l2(&self, req: CallRequest) -> RpcResult; + async fn estimate_gas_l1_to_l2( + &self, + req: CallRequest, + state_override: Option, + ) -> RpcResult; #[method(name = "getBridgehubContract")] async fn get_bridgehub_contract(&self) -> RpcResult>; diff --git a/core/node/api_server/src/execution_sandbox/execute.rs b/core/node/api_server/src/execution_sandbox/execute.rs index 75b226e6d8d6..f633b133ab00 100644 --- a/core/node/api_server/src/execution_sandbox/execute.rs +++ b/core/node/api_server/src/execution_sandbox/execute.rs @@ -173,6 +173,7 @@ impl TransactionExecutor { block_args: BlockArgs, vm_execution_cache_misses_limit: Option, custom_tracers: Vec, + state_override: Option, ) -> anyhow::Result { let execution_args = TxExecutionArgs::for_eth_call( call_overrides.enforced_base_fee, @@ -192,7 +193,7 @@ impl TransactionExecutor { connection_pool, tx.into(), block_args, - None, + state_override, custom_tracers, ) .await?; diff --git a/core/node/api_server/src/tx_sender/mod.rs b/core/node/api_server/src/tx_sender/mod.rs index 7302fd7e36a5..15f9271d6428 100644 --- a/core/node/api_server/src/tx_sender/mod.rs +++ b/core/node/api_server/src/tx_sender/mod.rs @@ -989,6 +989,7 @@ impl TxSender { block_args: BlockArgs, call_overrides: CallOverrides, tx: L2Tx, + state_override: Option, ) -> Result, SubmitTxError> { let vm_permit = self.0.vm_concurrency_limiter.acquire().await; let vm_permit = vm_permit.ok_or(SubmitTxError::ServerShuttingDown)?; @@ -1005,6 +1006,7 @@ impl TxSender { block_args, vm_execution_cache_misses_limit, vec![], + state_override, ) .await? .into_api_call_result() diff --git a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/eth.rs b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/eth.rs index b9c9cda7b858..ff8ce0356a05 100644 --- a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/eth.rs +++ b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/eth.rs @@ -27,8 +27,13 @@ impl EthNamespaceServer for EthNamespace { Ok(self.chain_id_impl()) } - async fn call(&self, req: CallRequest, block: Option) -> RpcResult { - self.call_impl(req, block.map(Into::into)) + async fn call( + &self, + req: CallRequest, + block: Option, + state_override: Option, + ) -> RpcResult { + self.call_impl(req, block.map(Into::into), state_override) .await .map_err(|err| self.current_method().map_err(err)) } diff --git a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs index 45cb312dde6e..16bbde13509f 100644 --- a/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs +++ b/core/node/api_server/src/web3/backend_jsonrpsee/namespaces/zks.rs @@ -3,8 +3,9 @@ use std::collections::HashMap; use itertools::Itertools; use zksync_types::{ api::{ - ApiStorageLog, BlockDetails, BridgeAddresses, L1BatchDetails, L2ToL1LogProof, Log, Proof, - ProtocolVersion, TransactionDetailedResult, TransactionDetails, + state_override::StateOverride, ApiStorageLog, BlockDetails, BridgeAddresses, + L1BatchDetails, L2ToL1LogProof, Log, Proof, ProtocolVersion, TransactionDetailedResult, + TransactionDetails, }, fee::Fee, fee_model::{FeeParams, PubdataIndependentBatchFeeModelInput}, @@ -22,14 +23,22 @@ use crate::web3::ZksNamespace; #[async_trait] impl ZksNamespaceServer for ZksNamespace { - async fn estimate_fee(&self, req: CallRequest) -> RpcResult { - self.estimate_fee_impl(req) + async fn estimate_fee( + &self, + req: CallRequest, + state_override: Option, + ) -> RpcResult { + self.estimate_fee_impl(req, state_override) .await .map_err(|err| self.current_method().map_err(err)) } - async fn estimate_gas_l1_to_l2(&self, req: CallRequest) -> RpcResult { - self.estimate_l1_to_l2_gas_impl(req) + async fn estimate_gas_l1_to_l2( + &self, + req: CallRequest, + state_override: Option, + ) -> RpcResult { + self.estimate_l1_to_l2_gas_impl(req, state_override) .await .map_err(|err| self.current_method().map_err(err)) } diff --git a/core/node/api_server/src/web3/namespaces/debug.rs b/core/node/api_server/src/web3/namespaces/debug.rs index a2e6e2782ac5..2f2d1d44cba1 100644 --- a/core/node/api_server/src/web3/namespaces/debug.rs +++ b/core/node/api_server/src/web3/namespaces/debug.rs @@ -197,6 +197,7 @@ impl DebugNamespace { block_args, self.sender_config().vm_execution_cache_misses_limit, custom_tracers, + None, ) .await?; diff --git a/core/node/api_server/src/web3/namespaces/eth.rs b/core/node/api_server/src/web3/namespaces/eth.rs index df46b894f4bd..68030763fd60 100644 --- a/core/node/api_server/src/web3/namespaces/eth.rs +++ b/core/node/api_server/src/web3/namespaces/eth.rs @@ -55,6 +55,7 @@ impl EthNamespace { &self, mut request: CallRequest, block_id: Option, + state_override: Option, ) -> Result { let block_id = block_id.unwrap_or(BlockId::Number(BlockNumber::Pending)); self.current_method().set_block_id(block_id); @@ -88,7 +89,7 @@ impl EthNamespace { let call_result: Vec = self .state .tx_sender - .eth_call(block_args, call_overrides, tx) + .eth_call(block_args, call_overrides, tx, state_override) .await?; Ok(call_result.into()) } diff --git a/core/node/api_server/src/web3/namespaces/zks.rs b/core/node/api_server/src/web3/namespaces/zks.rs index 25260741c76a..4f88eb17e231 100644 --- a/core/node/api_server/src/web3/namespaces/zks.rs +++ b/core/node/api_server/src/web3/namespaces/zks.rs @@ -8,8 +8,8 @@ use zksync_multivm::interface::VmExecutionResultAndLogs; use zksync_system_constants::DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE; use zksync_types::{ api::{ - BlockDetails, BridgeAddresses, GetLogsFilter, L1BatchDetails, L2ToL1LogProof, Proof, - ProtocolVersion, StorageProof, TransactionDetails, + state_override::StateOverride, BlockDetails, BridgeAddresses, GetLogsFilter, + L1BatchDetails, L2ToL1LogProof, Proof, ProtocolVersion, StorageProof, TransactionDetails, }, fee::Fee, fee_model::{FeeParams, PubdataIndependentBatchFeeModelInput}, @@ -48,7 +48,11 @@ impl ZksNamespace { &self.state.current_method } - pub async fn estimate_fee_impl(&self, request: CallRequest) -> Result { + pub async fn estimate_fee_impl( + &self, + request: CallRequest, + state_override: Option, + ) -> Result { let mut request_with_gas_per_pubdata_overridden = request; self.state .set_nonce_for_call_request(&mut request_with_gas_per_pubdata_overridden) @@ -67,12 +71,13 @@ impl ZksNamespace { // not consider provided ones. tx.common_data.fee.max_priority_fee_per_gas = 0u64.into(); tx.common_data.fee.gas_per_pubdata_limit = U256::from(DEFAULT_L2_TX_GAS_PER_PUBDATA_BYTE); - self.estimate_fee(tx.into()).await + self.estimate_fee(tx.into(), state_override).await } pub async fn estimate_l1_to_l2_gas_impl( &self, request: CallRequest, + state_override: Option, ) -> Result { let mut request_with_gas_per_pubdata_overridden = request; // When we're estimating fee, we are trying to deduce values related to fee, so we should @@ -87,11 +92,15 @@ impl ZksNamespace { .try_into() .map_err(Web3Error::SerializationError)?; - let fee = self.estimate_fee(tx.into()).await?; + let fee = self.estimate_fee(tx.into(), state_override).await?; Ok(fee.gas_limit) } - async fn estimate_fee(&self, tx: Transaction) -> Result { + async fn estimate_fee( + &self, + tx: Transaction, + state_override: Option, + ) -> Result { let scale_factor = self.state.api_config.estimate_gas_scale_factor; let acceptable_overestimation = self.state.api_config.estimate_gas_acceptable_overestimation; @@ -99,7 +108,12 @@ impl ZksNamespace { Ok(self .state .tx_sender - .get_txs_fee_in_wei(tx, scale_factor, acceptable_overestimation as u64, None) + .get_txs_fee_in_wei( + tx, + scale_factor, + acceptable_overestimation as u64, + state_override, + ) .await?) } diff --git a/core/node/api_server/src/web3/tests/vm.rs b/core/node/api_server/src/web3/tests/vm.rs index 4f4f5655797b..61c24bcf9001 100644 --- a/core/node/api_server/src/web3/tests/vm.rs +++ b/core/node/api_server/src/web3/tests/vm.rs @@ -64,7 +64,9 @@ impl HttpTest for CallTest { client: &DynClient, _pool: &ConnectionPool, ) -> anyhow::Result<()> { - let call_result = client.call(Self::call_request(b"pending"), None).await?; + let call_result = client + .call(Self::call_request(b"pending"), None, None) + .await?; assert_eq!(call_result.0, b"output"); let valid_block_numbers_and_calldata = [ @@ -75,7 +77,7 @@ impl HttpTest for CallTest { for (number, calldata) in valid_block_numbers_and_calldata { let number = api::BlockIdVariant::BlockNumber(number); let call_result = client - .call(Self::call_request(calldata), Some(number)) + .call(Self::call_request(calldata), Some(number), None) .await?; assert_eq!(call_result.0, b"output"); } @@ -83,7 +85,7 @@ impl HttpTest for CallTest { let invalid_block_number = api::BlockNumber::from(100); let number = api::BlockIdVariant::BlockNumber(invalid_block_number); let error = client - .call(Self::call_request(b"100"), Some(number)) + .call(Self::call_request(b"100"), Some(number), None) .await .unwrap_err(); if let ClientError::Call(error) = error { @@ -121,7 +123,7 @@ impl HttpTest for CallTestAfterSnapshotRecovery { _pool: &ConnectionPool, ) -> anyhow::Result<()> { let call_result = client - .call(CallTest::call_request(b"pending"), None) + .call(CallTest::call_request(b"pending"), None, None) .await?; assert_eq!(call_result.0, b"output"); let pending_block_number = api::BlockIdVariant::BlockNumber(api::BlockNumber::Pending); @@ -129,6 +131,7 @@ impl HttpTest for CallTestAfterSnapshotRecovery { .call( CallTest::call_request(b"pending"), Some(pending_block_number), + None, ) .await?; assert_eq!(call_result.0, b"output"); @@ -138,7 +141,7 @@ impl HttpTest for CallTestAfterSnapshotRecovery { for number in pruned_block_numbers { let number = api::BlockIdVariant::BlockNumber(number.into()); let error = client - .call(CallTest::call_request(b"pruned"), Some(number)) + .call(CallTest::call_request(b"pruned"), Some(number), None) .await .unwrap_err(); assert_pruned_block_error(&error, first_local_l2_block); @@ -148,7 +151,7 @@ impl HttpTest for CallTestAfterSnapshotRecovery { for number in first_l2_block_numbers { let number = api::BlockIdVariant::BlockNumber(number); let call_result = client - .call(CallTest::call_request(b"first"), Some(number)) + .call(CallTest::call_request(b"first"), Some(number), None) .await?; assert_eq!(call_result.0, b"output"); } @@ -500,7 +503,7 @@ impl HttpTest for TraceCallTestAfterSnapshotRecovery { for number in pruned_block_numbers { let number = api::BlockIdVariant::BlockNumber(number.into()); let error = client - .call(CallTest::call_request(b"pruned"), Some(number)) + .call(CallTest::call_request(b"pruned"), Some(number), None) .await .unwrap_err(); assert_pruned_block_error(&error, first_local_l2_block); diff --git a/core/tests/loadnext/src/sdk/operations/deploy_contract.rs b/core/tests/loadnext/src/sdk/operations/deploy_contract.rs index af621249ed8b..3b4c7a5eb53f 100644 --- a/core/tests/loadnext/src/sdk/operations/deploy_contract.rs +++ b/core/tests/loadnext/src/sdk/operations/deploy_contract.rs @@ -155,7 +155,7 @@ where ); self.wallet .provider - .estimate_fee(l2_tx.into()) + .estimate_fee(l2_tx.into(), None) .await .map_err(Into::into) } diff --git a/core/tests/loadnext/src/sdk/operations/execute_contract.rs b/core/tests/loadnext/src/sdk/operations/execute_contract.rs index 18b93008a73a..d5fe57c7b79f 100644 --- a/core/tests/loadnext/src/sdk/operations/execute_contract.rs +++ b/core/tests/loadnext/src/sdk/operations/execute_contract.rs @@ -155,7 +155,7 @@ where ); self.wallet .provider - .estimate_fee(execute.into()) + .estimate_fee(execute.into(), None) .await .map_err(Into::into) } diff --git a/core/tests/loadnext/src/sdk/operations/transfer.rs b/core/tests/loadnext/src/sdk/operations/transfer.rs index 34bab615c7c5..94ee3aeb6082 100644 --- a/core/tests/loadnext/src/sdk/operations/transfer.rs +++ b/core/tests/loadnext/src/sdk/operations/transfer.rs @@ -181,7 +181,7 @@ where }; self.wallet .provider - .estimate_fee(l2_tx.into()) + .estimate_fee(l2_tx.into(), None) .await .map_err(Into::into) } diff --git a/core/tests/loadnext/src/sdk/wallet.rs b/core/tests/loadnext/src/sdk/wallet.rs index c46431f70f48..9d3bd73a9bf2 100644 --- a/core/tests/loadnext/src/sdk/wallet.rs +++ b/core/tests/loadnext/src/sdk/wallet.rs @@ -96,7 +96,7 @@ where }; let bytes = self .provider - .call(req, Some(BlockIdVariant::BlockNumber(block_number))) + .call(req, Some(BlockIdVariant::BlockNumber(block_number)), None) .await?; if bytes.0.len() == 32 { U256::from_big_endian(&bytes.0) diff --git a/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol b/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol new file mode 100644 index 000000000000..05209007da9b --- /dev/null +++ b/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: MIT +pragma solidity ^0.8.0; + +pragma solidity ^0.8.0; + +contract StateOverrideTest { + uint256 public someValue; + uint256 public anotherValue; + + function setValue(uint256 value) public { + someValue = value; + } + + function setAnotherValue(uint256 value) public { + anotherValue = value; + } + + function increment(uint256 value) public view returns (uint256) { + require(someValue > 0, "Initial state not set"); + return someValue + value; + } + + function sumValues() public view returns (uint256) { + require(someValue > 0 && anotherValue > 0, "Initial state not set"); + return someValue + anotherValue; + } +} \ No newline at end of file diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index 0c67f3b1fbc2..1a2ec2beaf3d 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -20,7 +20,8 @@ const contracts = { counter: getTestContract('Counter'), events: getTestContract('Emitter'), outer: getTestContract('Outer'), - inner: getTestContract('Inner') + inner: getTestContract('Inner'), + stateOverride: getTestContract('StateOverrideTest') }; describe('web3 API compatibility tests', () => { @@ -945,10 +946,17 @@ describe('web3 API compatibility tests', () => { describe('Storage override', () => { test('Should be able to estimate_gas overriding the balance of the sender', async () => { - const balance = await alice.getBalance(l2Token); + const balance = await alice.getBalance(); const amount = balance.add(1); - // const tx = await alice.transfer({ to: alice.address, token: l2Token, amount }); - // await expect(tx).toBeRejected('insufficient balance for transfer'); + + // Expect the transaction to be reverted without the overridden balance + await expect( + alice.provider.estimateGas({ + from: alice.address, + to: alice.address, + value: amount.toHexString() + }) + ).toBeRejected(); // Call estimate_gas overriding the balance of the sender using the eth_estimateGas endpoint const response = await alice.provider.send('eth_estimateGas', [ @@ -1004,6 +1012,121 @@ describe('web3 API compatibility tests', () => { // Assert that the response is successful expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); }); + + test('Should estimate gas by overriding state with State', async () => { + const contract = await deployContract(alice, contracts.stateOverride, []); + + const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); + + // Ensure that the initial gas estimation fails due to contract requirements + await expect( + alice.provider.estimateGas({ + to: contract.address, + data: sumValuesFunctionData + }) + ).toBeRejected(); + + // Override the entire contract state using State + const state = { + [contract.address]: { + state: { + '0x0000000000000000000000000000000000000000000000000000000000000000': + '0x0000000000000000000000000000000000000000000000000000000000000001', + '0x0000000000000000000000000000000000000000000000000000000000000001': + '0x0000000000000000000000000000000000000000000000000000000000000002' + } + } + }; + + const response = await alice.provider.send('eth_estimateGas', [ + { + from: alice.address, + to: contract.address, + data: sumValuesFunctionData + }, + 'latest', + state + ]); + + expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); + }); + + test('Should estimate gas by overriding state with StateDiff', async () => { + const contract = await deployContract(alice, contracts.stateOverride, []); + const incrementFunctionData = contract.interface.encodeFunctionData('increment', [1]); + + // Ensure that the initial gas estimation fails due to contract requirements + await expect( + alice.provider.estimateGas({ + to: contract.address, + data: incrementFunctionData + }) + ).toBeRejected(); + + // Override the contract state using StateDiff + const stateDiff = { + [contract.address]: { + stateDiff: { + '0x0000000000000000000000000000000000000000000000000000000000000000': + '0x0000000000000000000000000000000000000000000000000000000000000001' + } + } + }; + + const response = await alice.provider.send('eth_estimateGas', [ + { + from: alice.address, + to: contract.address, + data: incrementFunctionData + }, + 'latest', + stateDiff + ]); + + expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); + }); + }); + + test('Should call and succeed with overriding state with State', async () => { + const contract = await deployContract(alice, contracts.stateOverride, []); + const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); + + // Ensure that the initial call fails due to contract requirements + const callResponse = await alice.provider.call({ + to: contract.address, + data: sumValuesFunctionData + }); + + const errorString = 'Initial state not set'; + const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); + const errorSelector = '0x08c379a0'; + const expectedResponse = errorSelector + encodedErrorString.slice(2); + + expect(callResponse).toEqual(expectedResponse); + + // Override the contract state using State + const state = { + [contract.address]: { + state: { + '0x0000000000000000000000000000000000000000000000000000000000000000': + '0x0000000000000000000000000000000000000000000000000000000000000001', + '0x0000000000000000000000000000000000000000000000000000000000000001': + '0x0000000000000000000000000000000000000000000000000000000000000002' + } + } + }; + + const response = await alice.provider.send('eth_call', [ + { + from: alice.address, + to: contract.address, + data: sumValuesFunctionData + }, + 'latest', + state + ]); + + expect(response).toEqual('0x0000000000000000000000000000000000000000000000000000000000000003'); }); // We want to be sure that correct(outer) contract address is return in the transaction receipt, From 9649f59b0d8dd7b0ec7eb7ebb1a78523fbce2a28 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Mon, 1 Jul 2024 16:11:51 -0300 Subject: [PATCH 22/26] remove toml --- core/lib/zksync_core/Cargo.toml | 107 -------------------------------- 1 file changed, 107 deletions(-) delete mode 100644 core/lib/zksync_core/Cargo.toml diff --git a/core/lib/zksync_core/Cargo.toml b/core/lib/zksync_core/Cargo.toml deleted file mode 100644 index d7db3fb4b9a3..000000000000 --- a/core/lib/zksync_core/Cargo.toml +++ /dev/null @@ -1,107 +0,0 @@ -[package] -name = "zksync_core" -version = "0.1.0" -edition = "2021" -authors = ["The Matter Labs Team "] -homepage = "https://zksync.io/" -repository = "https://github.com/matter-labs/zksync-era" -license = "MIT OR Apache-2.0" -keywords = ["blockchain", "zksync"] -categories = ["cryptography"] - -links = "zksync_core_proto" - -[dependencies] -vise = { git = "https://github.com/matter-labs/vise.git", version = "0.1.0", rev = "1c9cc500e92cf9ea052b230e114a6f9cce4fb2c1" } -zksync_state = { path = "../state" } -vm_utils = { path = "../vm_utils" } -zksync_types = { path = "../types" } -zksync_dal = { path = "../dal" } -zksync_config = { path = "../config" } -zksync_env_config = { path = "../env_config" } -zksync_protobuf_config = { path = "../protobuf_config" } -zksync_utils = { path = "../utils" } -zksync_contracts = { path = "../contracts" } -zksync_system_constants = { path = "../../lib/constants" } -zksync_commitment_utils = { path = "../commitment_utils" } -zksync_eth_client = { path = "../eth_client" } -zksync_eth_signer = { path = "../eth_signer" } -zksync_l1_contract_interface = { path = "../l1_contract_interface" } -zksync_mempool = { path = "../mempool" } -zksync_queued_job_processor = { path = "../queued_job_processor" } -zksync_circuit_breaker = { path = "../circuit_breaker" } -zksync_storage = { path = "../storage" } -zksync_merkle_tree = { path = "../merkle_tree" } -zksync_mini_merkle_tree = { path = "../mini_merkle_tree" } -prometheus_exporter = { path = "../prometheus_exporter" } -zksync_prover_interface = { path = "../prover_interface" } -zksync_web3_decl = { path = "../web3_decl", default-features = false, features = [ - "server", - "client", -] } -zksync_object_store = { path = "../object_store" } -zksync_health_check = { path = "../health_check" } -vlog = { path = "../vlog" } - -multivm = { path = "../multivm" } - -# Consensus dependenices -zksync_concurrency = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_crypto = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_network = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_roles = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_storage = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_executor = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_bft = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_consensus_utils = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } -zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } - -prost = "0.12.1" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" -serde_yaml = "0.9" -itertools = "0.10.3" -metrics = "0.21" -ctrlc = { version = "3.1", features = ["termination"] } -rand = "0.8" -rlp = "0.5" - -tokio = { version = "1", features = ["time"] } -futures = { version = "0.3", features = ["compat"] } -pin-project-lite = "0.2.13" -chrono = { version = "0.4", features = ["serde"] } -anyhow = "1.0" -thiserror = "1.0" -async-trait = "0.1" -bitflags = "1.3.2" -thread_local = "1.1" - -reqwest = { version = "0.11", features = ["blocking", "json"] } -hex = "0.4" -lru = { version = "0.12.1", default-features = false } -governor = "0.4.2" -tower-http = { version = "0.4.1", features = ["full"] } -tower = { version = "0.4.13", features = ["full"] } -axum = { version = "0.6.19", default-features = false, features = [ - "http1", - "json", - "tokio", -] } -once_cell = "1.7" - -actix-rt = "2.2.0" -actix-cors = "0.6.0-beta.2" -actix-web = "4.0.0-beta.8" - -tracing = "0.1.26" - -[dev-dependencies] -zksync_test_account = { path = "../../tests/test_account" } - -assert_matches = "1.5" -jsonrpsee = "0.21.0" -tempfile = "3.0.2" -test-casing = "0.1.2" - -[build-dependencies] -zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "842d4fd79f1d7dae946b6873ded7ad391d554814" } From 5cb963e6bf1e66a7bd7b834f0d9112be25b73dfb Mon Sep 17 00:00:00 2001 From: Jrigada Date: Tue, 2 Jul 2024 09:49:18 -0300 Subject: [PATCH 23/26] outdated comment --- core/lib/types/src/api/state_override.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/lib/types/src/api/state_override.rs b/core/lib/types/src/api/state_override.rs index 4ea60d7b0459..5c2395ae4bf2 100644 --- a/core/lib/types/src/api/state_override.rs +++ b/core/lib/types/src/api/state_override.rs @@ -5,7 +5,7 @@ use zksync_basic_types::{web3::Bytes, H256, U256}; use crate::Address; -/// Collection of overridden accounts, useful for `eth_estimateGas`. +/// Collection of overridden accounts #[derive(Debug, Clone, Serialize, Deserialize)] pub struct StateOverride(HashMap); From a0627cb96e5e18730b1ab9f5272d1d0b51de526a Mon Sep 17 00:00:00 2001 From: Jrigada Date: Thu, 11 Jul 2024 10:19:42 -0300 Subject: [PATCH 24/26] add logic for testing stateDiff and State differences and fix nits --- core/lib/state/src/storage_overrides.rs | 12 +- .../state-override/StateOverrideTest.sol | 5 +- .../ts-integration/tests/api/web3.test.ts | 118 ++++++++++++------ 3 files changed, 93 insertions(+), 42 deletions(-) diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index f6a45d6a6484..30637e29a6c2 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -16,7 +16,7 @@ pub struct StorageOverrides { storage_handle: S, overridden_factory_deps: HashMap>, overridden_account_state: HashMap>, - overriden_account_state_diff: HashMap>, + overridden_account_state_diff: HashMap>, overridden_balance: HashMap, overridden_nonce: HashMap, overridden_code: HashMap, @@ -29,7 +29,7 @@ impl StorageOverrides { storage_handle: storage, overridden_factory_deps: HashMap::new(), overridden_account_state: HashMap::new(), - overriden_account_state_diff: HashMap::new(), + overridden_account_state_diff: HashMap::new(), overridden_balance: HashMap::new(), overridden_nonce: HashMap::new(), overridden_code: HashMap::new(), @@ -52,7 +52,7 @@ impl StorageOverrides { account: AccountTreeId, state_diff: HashMap, ) { - self.overriden_account_state_diff + self.overridden_account_state_diff .insert(account, state_diff); } @@ -78,10 +78,12 @@ impl ReadStorage for StorageOverrides { if let Some(account_state) = self.overridden_account_state.get(key.account()) { if let Some(value) = account_state.get(key.key()) { return *value; + } else { + return H256::zero(); } } - if let Some(account_state_diff) = self.overriden_account_state_diff.get(key.account()) { + if let Some(account_state_diff) = self.overridden_account_state_diff.get(key.account()) { if let Some(value) = account_state_diff.get(key.key()) { return *value; } @@ -136,7 +138,7 @@ impl OverrideStorage for StorageOverrides { Some(OverrideState::StateDiff(state_diff)) => { for (key, value) in state_diff { let account_state = self - .overridden_account_state + .overridden_account_state_diff .entry(AccountTreeId::new(*account)) .or_default(); account_state.insert(*key, *value); diff --git a/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol b/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol index 05209007da9b..e8d02737cc15 100644 --- a/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol +++ b/core/tests/ts-integration/contracts/state-override/StateOverrideTest.sol @@ -6,6 +6,7 @@ pragma solidity ^0.8.0; contract StateOverrideTest { uint256 public someValue; uint256 public anotherValue; + uint256 public initialValue = 100; function setValue(uint256 value) public { someValue = value; @@ -22,6 +23,6 @@ contract StateOverrideTest { function sumValues() public view returns (uint256) { require(someValue > 0 && anotherValue > 0, "Initial state not set"); - return someValue + anotherValue; + return someValue + anotherValue + initialValue; } -} \ No newline at end of file +} diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index 1a2ec2beaf3d..719500570c8d 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -1085,50 +1085,98 @@ describe('web3 API compatibility tests', () => { expect(response).toEqual(expect.stringMatching(HEX_VALUE_REGEX)); }); - }); - test('Should call and succeed with overriding state with State', async () => { - const contract = await deployContract(alice, contracts.stateOverride, []); - const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); + test('Should call and succeed with overriding state with State', async () => { + const contract = await deployContract(alice, contracts.stateOverride, []); + const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); - // Ensure that the initial call fails due to contract requirements - const callResponse = await alice.provider.call({ - to: contract.address, - data: sumValuesFunctionData - }); + // Ensure that the initial call fails due to contract requirements + const callResponse = await alice.provider.call({ + to: contract.address, + data: sumValuesFunctionData + }); + + const errorString = 'Initial state not set'; + const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); + const errorSelector = '0x08c379a0'; + const expectedResponse = errorSelector + encodedErrorString.slice(2); - const errorString = 'Initial state not set'; - const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); - const errorSelector = '0x08c379a0'; - const expectedResponse = errorSelector + encodedErrorString.slice(2); - - expect(callResponse).toEqual(expectedResponse); - - // Override the contract state using State - const state = { - [contract.address]: { - state: { - '0x0000000000000000000000000000000000000000000000000000000000000000': - '0x0000000000000000000000000000000000000000000000000000000000000001', - '0x0000000000000000000000000000000000000000000000000000000000000001': - '0x0000000000000000000000000000000000000000000000000000000000000002' + expect(callResponse).toEqual(expectedResponse); + + // Override the contract state using State + const state = { + [contract.address]: { + state: { + '0x0000000000000000000000000000000000000000000000000000000000000000': + '0x0000000000000000000000000000000000000000000000000000000000000001', + '0x0000000000000000000000000000000000000000000000000000000000000001': + '0x0000000000000000000000000000000000000000000000000000000000000002' + } } - } - }; + }; - const response = await alice.provider.send('eth_call', [ - { - from: alice.address, + const response = await alice.provider.send('eth_call', [ + { + from: alice.address, + to: contract.address, + data: sumValuesFunctionData + }, + 'latest', + state + ]); + + // The state replace the entire state of the contract, so the sum now would be + // 1 (0x1) + 2 (0x2) = 3 (0x3) + expect(response).toEqual('0x0000000000000000000000000000000000000000000000000000000000000003'); + }); + + test('Should call and succeed with overriding state with StateDiff', async () => { + const contract = await deployContract(alice, contracts.stateOverride, []); + const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); + + // Ensure that the initial call fails due to contract requirements + const callResponse = await alice.provider.call({ to: contract.address, data: sumValuesFunctionData - }, - 'latest', - state - ]); + }); - expect(response).toEqual('0x0000000000000000000000000000000000000000000000000000000000000003'); - }); + const errorString = 'Initial state not set'; + const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); + const errorSelector = '0x08c379a0'; + const expectedResponse = errorSelector + encodedErrorString.slice(2); + expect(callResponse).toEqual(expectedResponse); + + // Override the contract state using State + const stateDiff = { + [contract.address]: { + stateDiff: { + '0x0000000000000000000000000000000000000000000000000000000000000000': + '0x0000000000000000000000000000000000000000000000000000000000000001', + '0x0000000000000000000000000000000000000000000000000000000000000001': + '0x0000000000000000000000000000000000000000000000000000000000000002' + } + } + }; + + const response = await alice.provider.send('eth_call', [ + { + from: alice.address, + to: contract.address, + data: sumValuesFunctionData + }, + 'latest', + stateDiff + ]); + + // The stateDiff only changes the specific slots provided in the override. + // The initial value of the storage slot at key 0x2 remains unchanged, which is 100 (0x64 in hex). + // Therefore, the sum of the values at the three storage slots is: + // 1 (0x1) + 2 (0x2) + 100 (0x64) = 103 (0x67 in hex). + // This is why the expected response is 0x67. + expect(response).toEqual('0x0000000000000000000000000000000000000000000000000000000000000067'); + }); + }); // We want to be sure that correct(outer) contract address is return in the transaction receipt, // when there is a contract that initializa another contract in the constructor test('Should check inner-outer contract address in the receipt of the deploy tx', async () => { From cf7b8cbbc937ec776d8d873b8c56cdc2b588ae56 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Fri, 12 Jul 2024 10:15:34 -0300 Subject: [PATCH 25/26] fix error on merge --- core/lib/state/src/lib.rs | 1 + core/lib/state/src/storage_overrides.rs | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/core/lib/state/src/lib.rs b/core/lib/state/src/lib.rs index 197089b8049c..74c60e4a3695 100644 --- a/core/lib/state/src/lib.rs +++ b/core/lib/state/src/lib.rs @@ -42,6 +42,7 @@ mod postgres; mod rocksdb; mod shadow_storage; mod storage_factory; +mod storage_overrides; mod storage_view; #[cfg(test)] mod test_utils; diff --git a/core/lib/state/src/storage_overrides.rs b/core/lib/state/src/storage_overrides.rs index 30637e29a6c2..f45dd6d3382f 100644 --- a/core/lib/state/src/storage_overrides.rs +++ b/core/lib/state/src/storage_overrides.rs @@ -78,9 +78,8 @@ impl ReadStorage for StorageOverrides { if let Some(account_state) = self.overridden_account_state.get(key.account()) { if let Some(value) = account_state.get(key.key()) { return *value; - } else { - return H256::zero(); } + return H256::zero(); } if let Some(account_state_diff) = self.overridden_account_state_diff.get(key.account()) { From 3685e543015aeac7abf7c912ed5b0a48ab794a86 Mon Sep 17 00:00:00 2001 From: Jrigada Date: Wed, 17 Jul 2024 10:35:01 -0300 Subject: [PATCH 26/26] fix tests --- .../ts-integration/tests/api/web3.test.ts | 82 ++++++++++--------- 1 file changed, 42 insertions(+), 40 deletions(-) diff --git a/core/tests/ts-integration/tests/api/web3.test.ts b/core/tests/ts-integration/tests/api/web3.test.ts index a6b6b327c510..e78ec452b2f5 100644 --- a/core/tests/ts-integration/tests/api/web3.test.ts +++ b/core/tests/ts-integration/tests/api/web3.test.ts @@ -972,14 +972,14 @@ describe('web3 API compatibility tests', () => { describe('Storage override', () => { test('Should be able to estimate_gas overriding the balance of the sender', async () => { const balance = await alice.getBalance(); - const amount = balance.add(1); + const amount = balance + 1n; // Expect the transaction to be reverted without the overridden balance await expect( alice.provider.estimateGas({ from: alice.address, to: alice.address, - value: amount.toHexString() + value: amount.toString() }) ).toBeRejected(); @@ -988,13 +988,13 @@ describe('web3 API compatibility tests', () => { { from: alice.address, to: alice.address, - value: amount.toHexString() + value: amount.toString() }, 'latest', //override with the balance needed to send the transaction { [alice.address]: { - balance: amount.toHexString() + balance: amount.toString() } } ]); @@ -1005,12 +1005,14 @@ describe('web3 API compatibility tests', () => { test('Should be able to estimate_gas overriding contract code', async () => { // Deploy the first contract const contract1 = await deployContract(alice, contracts.events, []); + const contract1Address = await contract1.getAddress(); // Deploy the second contract to extract the code that we are overriding the estimation with const contract2 = await deployContract(alice, contracts.counter, []); + const contract2Address = await contract2.getAddress(); // Get the code of contract2 - const code = await alice.provider.getCode(contract2.address); + const code = await alice.provider.getCode(contract2Address); // Get the calldata of the increment function of contract2 const incrementFunctionData = contract2.interface.encodeFunctionData('increment', [1]); @@ -1018,7 +1020,7 @@ describe('web3 API compatibility tests', () => { // Assert that the estimation fails because the increment function is not present in contract1 expect( alice.provider.estimateGas({ - to: contract1.address, + to: contract1Address.toString(), data: incrementFunctionData }) ).toBeRejected(); @@ -1027,11 +1029,11 @@ describe('web3 API compatibility tests', () => { const response = await alice.provider.send('eth_estimateGas', [ { from: alice.address, - to: contract1.address, + to: contract1Address.toString(), data: incrementFunctionData }, 'latest', - { [contract1.address]: { code: code } } + { [contract1Address.toString()]: { code: code } } ]); // Assert that the response is successful @@ -1040,20 +1042,21 @@ describe('web3 API compatibility tests', () => { test('Should estimate gas by overriding state with State', async () => { const contract = await deployContract(alice, contracts.stateOverride, []); + const contractAddress = await contract.getAddress(); const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); // Ensure that the initial gas estimation fails due to contract requirements await expect( alice.provider.estimateGas({ - to: contract.address, + to: contractAddress.toString(), data: sumValuesFunctionData }) ).toBeRejected(); // Override the entire contract state using State const state = { - [contract.address]: { + [contractAddress.toString()]: { state: { '0x0000000000000000000000000000000000000000000000000000000000000000': '0x0000000000000000000000000000000000000000000000000000000000000001', @@ -1066,7 +1069,7 @@ describe('web3 API compatibility tests', () => { const response = await alice.provider.send('eth_estimateGas', [ { from: alice.address, - to: contract.address, + to: contractAddress.toString(), data: sumValuesFunctionData }, 'latest', @@ -1078,19 +1081,20 @@ describe('web3 API compatibility tests', () => { test('Should estimate gas by overriding state with StateDiff', async () => { const contract = await deployContract(alice, contracts.stateOverride, []); + const contractAddress = await contract.getAddress(); const incrementFunctionData = contract.interface.encodeFunctionData('increment', [1]); // Ensure that the initial gas estimation fails due to contract requirements await expect( alice.provider.estimateGas({ - to: contract.address, + to: contractAddress.toString(), data: incrementFunctionData }) ).toBeRejected(); // Override the contract state using StateDiff const stateDiff = { - [contract.address]: { + [contractAddress.toString()]: { stateDiff: { '0x0000000000000000000000000000000000000000000000000000000000000000': '0x0000000000000000000000000000000000000000000000000000000000000001' @@ -1101,7 +1105,7 @@ describe('web3 API compatibility tests', () => { const response = await alice.provider.send('eth_estimateGas', [ { from: alice.address, - to: contract.address, + to: contractAddress.toString(), data: incrementFunctionData }, 'latest', @@ -1113,24 +1117,23 @@ describe('web3 API compatibility tests', () => { test('Should call and succeed with overriding state with State', async () => { const contract = await deployContract(alice, contracts.stateOverride, []); + const contractAddress = await contract.getAddress(); const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); // Ensure that the initial call fails due to contract requirements - const callResponse = await alice.provider.call({ - to: contract.address, - data: sumValuesFunctionData - }); - - const errorString = 'Initial state not set'; - const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); - const errorSelector = '0x08c379a0'; - const expectedResponse = errorSelector + encodedErrorString.slice(2); - - expect(callResponse).toEqual(expectedResponse); + await alice.provider + .call({ + to: contractAddress.toString(), + data: sumValuesFunctionData + }) + .catch((error) => { + const errorString = 'Initial state not set'; + expect(error.message).toContain(errorString); + }); // Override the contract state using State const state = { - [contract.address]: { + [contractAddress.toString()]: { state: { '0x0000000000000000000000000000000000000000000000000000000000000000': '0x0000000000000000000000000000000000000000000000000000000000000001', @@ -1143,7 +1146,7 @@ describe('web3 API compatibility tests', () => { const response = await alice.provider.send('eth_call', [ { from: alice.address, - to: contract.address, + to: contractAddress.toString(), data: sumValuesFunctionData }, 'latest', @@ -1157,24 +1160,23 @@ describe('web3 API compatibility tests', () => { test('Should call and succeed with overriding state with StateDiff', async () => { const contract = await deployContract(alice, contracts.stateOverride, []); + const contractAddress = await contract.getAddress(); const sumValuesFunctionData = contract.interface.encodeFunctionData('sumValues', []); // Ensure that the initial call fails due to contract requirements - const callResponse = await alice.provider.call({ - to: contract.address, - data: sumValuesFunctionData - }); - - const errorString = 'Initial state not set'; - const encodedErrorString = ethers.utils.defaultAbiCoder.encode(['string'], [errorString]); - const errorSelector = '0x08c379a0'; - const expectedResponse = errorSelector + encodedErrorString.slice(2); - - expect(callResponse).toEqual(expectedResponse); + await alice.provider + .call({ + to: contractAddress.toString(), + data: sumValuesFunctionData + }) + .catch((error) => { + const errorString = 'Initial state not set'; + expect(error.message).toContain(errorString); + }); // Override the contract state using State const stateDiff = { - [contract.address]: { + [contractAddress.toString()]: { stateDiff: { '0x0000000000000000000000000000000000000000000000000000000000000000': '0x0000000000000000000000000000000000000000000000000000000000000001', @@ -1187,7 +1189,7 @@ describe('web3 API compatibility tests', () => { const response = await alice.provider.send('eth_call', [ { from: alice.address, - to: contract.address, + to: contractAddress.toString(), data: sumValuesFunctionData }, 'latest',