diff --git a/Cargo.lock b/Cargo.lock index 36e7025cd55d..8e463524d3bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8199,6 +8199,7 @@ dependencies = [ "tempfile", "thiserror", "tracing", + "tracing-subscriber", "vise", "zksync_crypto", "zksync_storage", diff --git a/core/lib/merkle_tree/Cargo.toml b/core/lib/merkle_tree/Cargo.toml index 1204bdf6c945..b03fdeb697bd 100644 --- a/core/lib/merkle_tree/Cargo.toml +++ b/core/lib/merkle_tree/Cargo.toml @@ -32,3 +32,4 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" serde_with = { version = "1", features = ["hex"] } tempfile = "3.0.2" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } diff --git a/core/lib/merkle_tree/examples/loadtest/main.rs b/core/lib/merkle_tree/examples/loadtest/main.rs index 75971fd26fbc..89e14754bdb7 100644 --- a/core/lib/merkle_tree/examples/loadtest/main.rs +++ b/core/lib/merkle_tree/examples/loadtest/main.rs @@ -6,6 +6,7 @@ use clap::Parser; use rand::{rngs::StdRng, seq::IteratorRandom, SeedableRng}; use tempfile::TempDir; +use tracing_subscriber::EnvFilter; use std::{ thread, @@ -66,8 +67,16 @@ struct Cli { } impl Cli { + fn init_logging() { + tracing_subscriber::fmt() + .pretty() + .with_env_filter(EnvFilter::from_default_env()) + .init(); + } + fn run(self) { - println!("Launched with options: {self:?}"); + Self::init_logging(); + tracing::info!("Launched with options: {self:?}"); let (mut mock_db, mut rocksdb); let mut _temp_dir = None; @@ -77,7 +86,7 @@ impl Cli { &mut mock_db } else { let dir = TempDir::new().expect("failed creating temp dir for RocksDB"); - println!( + tracing::info!( "Created temp dir for RocksDB: {}", dir.path().to_string_lossy() ); @@ -127,7 +136,7 @@ impl Cli { let updated_keys = Self::generate_keys(updated_indices.into_iter()); let kvs = new_keys.into_iter().chain(updated_keys).zip(values); - println!("Processing block #{version}"); + tracing::info!("Processing block #{version}"); let start = Instant::now(); let root_hash = if self.proofs { let reads = Self::generate_keys(read_indices.into_iter()) @@ -143,15 +152,15 @@ impl Cli { output.root_hash }; let elapsed = start.elapsed(); - println!("Processed block #{version} in {elapsed:?}, root hash = {root_hash:?}"); + tracing::info!("Processed block #{version} in {elapsed:?}, root hash = {root_hash:?}"); } - println!("Verifying tree consistency..."); + tracing::info!("Verifying tree consistency..."); let start = Instant::now(); tree.verify_consistency(self.commit_count - 1) .expect("tree consistency check failed"); let elapsed = start.elapsed(); - println!("Verified tree consistency in {elapsed:?}"); + tracing::info!("Verified tree consistency in {elapsed:?}"); if let Some((pruner_handle, pruner_thread)) = pruner_handles { pruner_handle.abort(); @@ -170,5 +179,5 @@ impl Cli { } fn main() { - Cli::parse().run() + Cli::parse().run(); } diff --git a/core/lib/merkle_tree/examples/recovery.rs b/core/lib/merkle_tree/examples/recovery.rs new file mode 100644 index 000000000000..257944046047 --- /dev/null +++ b/core/lib/merkle_tree/examples/recovery.rs @@ -0,0 +1,120 @@ +//! Tree recovery load test. + +use clap::Parser; +use rand::{rngs::StdRng, Rng, SeedableRng}; +use tempfile::TempDir; +use tracing_subscriber::EnvFilter; + +use std::time::Instant; + +use zksync_crypto::hasher::blake2::Blake2Hasher; +use zksync_merkle_tree::{ + recovery::{MerkleTreeRecovery, RecoveryEntry}, + HashTree, Key, PatchSet, PruneDatabase, RocksDBWrapper, ValueHash, +}; +use zksync_storage::RocksDB; + +/// CLI for load-testing Merkle tree recovery. +#[derive(Debug, Parser)] +struct Cli { + /// Number of updates to perform. + #[arg(name = "updates")] + update_count: u64, + /// Number of entries per update. + #[arg(name = "ops")] + writes_per_update: usize, + /// Use a no-op hashing function. + #[arg(name = "no-hash", long)] + no_hashing: bool, + /// Perform testing on in-memory DB rather than RocksDB (i.e., with focus on hashing logic). + #[arg(long = "in-memory", short = 'M')] + in_memory: bool, + /// Block cache capacity for RocksDB in bytes. + #[arg(long = "block-cache", conflicts_with = "in_memory")] + block_cache: Option, + /// Seed to use in the RNG for reproducibility. + #[arg(long = "rng-seed", default_value = "0")] + rng_seed: u64, +} + +impl Cli { + fn init_logging() { + tracing_subscriber::fmt() + .pretty() + .with_env_filter(EnvFilter::from_default_env()) + .init(); + } + + fn run(self) { + Self::init_logging(); + tracing::info!("Launched with options: {self:?}"); + + let (mut mock_db, mut rocksdb); + let mut _temp_dir = None; + let db: &mut dyn PruneDatabase = if self.in_memory { + mock_db = PatchSet::default(); + &mut mock_db + } else { + let dir = TempDir::new().expect("failed creating temp dir for RocksDB"); + tracing::info!( + "Created temp dir for RocksDB: {}", + dir.path().to_string_lossy() + ); + rocksdb = if let Some(block_cache_capacity) = self.block_cache { + let db = RocksDB::with_cache(dir.path(), Some(block_cache_capacity)); + RocksDBWrapper::from(db) + } else { + RocksDBWrapper::new(dir.path()) + }; + _temp_dir = Some(dir); + &mut rocksdb + }; + + let hasher: &dyn HashTree = if self.no_hashing { &() } else { &Blake2Hasher }; + let mut rng = StdRng::seed_from_u64(self.rng_seed); + + let recovered_version = 123; + let key_step = + Key::MAX / (Key::from(self.update_count) * Key::from(self.writes_per_update)); + assert!(key_step > Key::from(u64::MAX)); + // ^ Total number of generated keys is <2^128. + + let mut last_key = Key::zero(); + let mut last_leaf_index = 0; + let mut recovery = MerkleTreeRecovery::with_hasher(db, recovered_version, hasher); + let recovery_started_at = Instant::now(); + for updated_idx in 0..self.update_count { + let started_at = Instant::now(); + let recovery_entries = (0..self.writes_per_update) + .map(|_| { + last_key += key_step - Key::from(rng.gen::()); + // ^ Increases the key by a random increment close to `key` step with some randomness. + last_leaf_index += 1; + RecoveryEntry { + key: last_key, + value: ValueHash::zero(), + leaf_index: last_leaf_index, + } + }) + .collect(); + recovery.extend(recovery_entries); + tracing::info!( + "Updated tree with recovery chunk #{updated_idx} in {:?}", + started_at.elapsed() + ); + } + + let tree = recovery.finalize(); + tracing::info!( + "Recovery finished in {:?}; verifying consistency...", + recovery_started_at.elapsed() + ); + let started_at = Instant::now(); + tree.verify_consistency(recovered_version).unwrap(); + tracing::info!("Verified consistency in {:?}", started_at.elapsed()); + } +} + +fn main() { + Cli::parse().run(); +} diff --git a/core/lib/merkle_tree/src/consistency.rs b/core/lib/merkle_tree/src/consistency.rs index 5a7e19202062..2cbe1691b39a 100644 --- a/core/lib/merkle_tree/src/consistency.rs +++ b/core/lib/merkle_tree/src/consistency.rs @@ -55,6 +55,14 @@ pub enum ConsistencyError { }, #[error("leaf with key {full_key} has same index {index} as another key")] DuplicateLeafIndex { index: u64, full_key: Key }, + #[error("internal node with key {key} does not have children")] + EmptyInternalNode { key: NodeKey }, + #[error( + "internal node with key {key} should have version {expected_version} (max among child ref versions)" + )] + KeyVersionMismatch { key: NodeKey, expected_version: u64 }, + #[error("root node should have version >={max_child_version} (max among child ref versions)")] + RootVersionMismatch { max_child_version: u64 }, } impl MerkleTree<'_, DB> @@ -109,6 +117,21 @@ where } Node::Internal(node) => { + let expected_version = node.child_refs().map(|child_ref| child_ref.version).max(); + let Some(expected_version) = expected_version else { + return Err(ConsistencyError::EmptyInternalNode { key }); + }; + if !key.is_empty() && expected_version != key.version { + return Err(ConsistencyError::KeyVersionMismatch { + key, + expected_version, + }); + } else if key.is_empty() && expected_version > key.version { + return Err(ConsistencyError::RootVersionMismatch { + max_child_version: expected_version, + }); + } + // `.into_par_iter()` below is the only place where `rayon`-based parallelism // is used in tree verification. let children: Vec<_> = node.children().collect(); @@ -239,7 +262,7 @@ mod tests { use std::num::NonZeroU64; use super::*; - use crate::PatchSet; + use crate::{types::InternalNode, PatchSet}; use zksync_types::{H256, U256}; const FIRST_KEY: Key = U256([0, 0, 0, 0x_dead_beef_0000_0000]); @@ -284,7 +307,7 @@ mod tests { #[test] fn missing_root_error() { let mut db = prepare_database(); - db.roots_mut().remove(&0); + db.remove_root(0); let err = MerkleTree::new(db).verify_consistency(0).unwrap_err(); assert_matches!(err, ConsistencyError::MissingRoot(0)); @@ -311,7 +334,7 @@ mod tests { fn leaf_count_mismatch_error() { let mut db = prepare_database(); - let root = db.roots_mut().get_mut(&0).unwrap(); + let root = db.root_mut(0).unwrap(); let Root::Filled { leaf_count, .. } = root else { panic!("unexpected root: {root:?}"); }; @@ -331,7 +354,7 @@ mod tests { fn hash_mismatch_error() { let mut db = prepare_database(); - let root = db.roots_mut().get_mut(&0).unwrap(); + let root = db.root_mut(0).unwrap(); let Root::Filled { node: Node::Internal(node), .. @@ -412,4 +435,62 @@ mod tests { let err = MerkleTree::new(db).verify_consistency(0).unwrap_err(); assert_matches!(err, ConsistencyError::DuplicateLeafIndex { index: 1, .. }); } + + #[test] + fn empty_internal_node_error() { + let mut db = prepare_database(); + let node_key = db.nodes_mut().find_map(|(key, node)| { + if let Node::Internal(node) = node { + *node = InternalNode::default(); + return Some(*key); + } + None + }); + let node_key = node_key.unwrap(); + + let err = MerkleTree::new(db).verify_consistency(0).unwrap_err(); + assert_matches!(err, ConsistencyError::EmptyInternalNode { key } if key == node_key); + } + + #[test] + fn version_mismatch_error() { + let mut db = prepare_database(); + let node_key = db.nodes_mut().find_map(|(key, node)| { + if let Node::Internal(node) = node { + let (nibble, _) = node.children().next().unwrap(); + node.child_ref_mut(nibble).unwrap().version = 1; + return Some(*key); + } + None + }); + let node_key = node_key.unwrap(); + + let err = MerkleTree::new(db).verify_consistency(0).unwrap_err(); + assert_matches!( + err, + ConsistencyError::KeyVersionMismatch { key, expected_version: 1 } if key == node_key + ); + } + + #[test] + fn root_version_mismatch_error() { + let mut db = prepare_database(); + let Some(Root::Filled { + node: Node::Internal(node), + .. + }) = db.root_mut(0) + else { + unreachable!(); + }; + let (nibble, _) = node.children().next().unwrap(); + node.child_ref_mut(nibble).unwrap().version = 42; + + let err = MerkleTree::new(db).verify_consistency(0).unwrap_err(); + assert_matches!( + err, + ConsistencyError::RootVersionMismatch { + max_child_version: 42, + } + ); + } } diff --git a/core/lib/merkle_tree/src/lib.rs b/core/lib/merkle_tree/src/lib.rs index a3344d1d6704..07a9668a61ae 100644 --- a/core/lib/merkle_tree/src/lib.rs +++ b/core/lib/merkle_tree/src/lib.rs @@ -48,6 +48,7 @@ mod getters; mod hasher; mod metrics; mod pruning; +pub mod recovery; mod storage; mod types; mod utils; @@ -146,7 +147,7 @@ impl<'a, DB: Database> MerkleTree<'a, DB> { pub fn with_hasher(db: DB, hasher: &'a dyn HashTree) -> Self { let tags = db.manifest().and_then(|manifest| manifest.tags); if let Some(tags) = tags { - tags.assert_consistency(hasher); + tags.assert_consistency(hasher, false); } // If there are currently no tags in the tree, we consider that it fits // for backward compatibility. The tags will be added the next time the tree is saved. @@ -208,7 +209,7 @@ impl<'a, DB: Database> MerkleTree<'a, DB> { /// Returns information about the update such as the final tree hash. pub fn extend(&mut self, key_value_pairs: Vec<(Key, ValueHash)>) -> BlockOutput { let next_version = self.db.manifest().unwrap_or_default().version_count; - let storage = Storage::new(&self.db, self.hasher, next_version); + let storage = Storage::new(&self.db, self.hasher, next_version, true); let (output, patch) = storage.extend(key_value_pairs); self.db.apply_patch(patch); output @@ -226,7 +227,7 @@ impl<'a, DB: Database> MerkleTree<'a, DB> { instructions: Vec<(Key, TreeInstruction)>, ) -> BlockOutputWithProofs { let next_version = self.db.manifest().unwrap_or_default().version_count; - let storage = Storage::new(&self.db, self.hasher, next_version); + let storage = Storage::new(&self.db, self.hasher, next_version, true); let (output, patch) = storage.extend_with_proofs(instructions); self.db.apply_patch(patch); output @@ -246,6 +247,7 @@ mod tests { architecture: "AR64MT".to_owned(), depth: 256, hasher: "blake2s256".to_string(), + is_recovering: false, }); MerkleTree::new(db); @@ -259,6 +261,7 @@ mod tests { architecture: "AR16MT".to_owned(), depth: 128, hasher: "blake2s256".to_string(), + is_recovering: false, }); MerkleTree::new(db); @@ -272,6 +275,7 @@ mod tests { architecture: "AR16MT".to_owned(), depth: 256, hasher: "sha256".to_string(), + is_recovering: false, }); MerkleTree::new(db); diff --git a/core/lib/merkle_tree/src/metrics.rs b/core/lib/merkle_tree/src/metrics.rs index 4e7b41bff9be..29bd58e599eb 100644 --- a/core/lib/merkle_tree/src/metrics.rs +++ b/core/lib/merkle_tree/src/metrics.rs @@ -7,7 +7,9 @@ use std::{ }; use crate::types::Nibbles; -use vise::{Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Global, Histogram, Metrics}; +use vise::{ + Buckets, EncodeLabelSet, EncodeLabelValue, Family, Gauge, Global, Histogram, Metrics, Unit, +}; #[derive(Debug, Metrics)] #[metrics(prefix = "merkle_tree")] @@ -24,19 +26,20 @@ const BYTE_SIZE_BUCKETS: Buckets = Buckets::exponential(65_536.0..=16.0 * 1_024. #[derive(Debug, Metrics)] #[metrics(prefix = "merkle_tree_finalize_patch")] struct HashingMetrics { - /// Total amount of hashing input performed while processing a single block. - #[metrics(buckets = BYTE_SIZE_BUCKETS)] - hashed_bytes: Histogram, + /// Total amount of hashing input performed while processing a patch. + #[metrics(buckets = BYTE_SIZE_BUCKETS, unit = Unit::Bytes)] + hashed: Histogram, + /// Total time spent on hashing while processing a patch. + #[metrics(buckets = Buckets::LATENCIES, unit = Unit::Seconds)] + hashing_duration: Histogram, } -#[vise::register] -static HASHING_METRICS: Global = Global::new(); - /// Hashing-related statistics reported as metrics for each block of operations. #[derive(Debug, Default)] #[must_use = "hashing stats should be `report()`ed"] pub(crate) struct HashingStats { pub hashed_bytes: AtomicU64, + pub hashing_duration: Duration, } impl HashingStats { @@ -45,8 +48,14 @@ impl HashingStats { } pub fn report(self) { + #[vise::register] + static HASHING_METRICS: Global = Global::new(); + let hashed_bytes = self.hashed_bytes.into_inner(); - HASHING_METRICS.hashed_bytes.observe(hashed_bytes); + HASHING_METRICS.hashed.observe(hashed_bytes); + HASHING_METRICS + .hashing_duration + .observe(self.hashing_duration); } } @@ -96,7 +105,7 @@ struct TreeUpdateMetrics { static TREE_UPDATE_METRICS: Global = Global::new(); #[must_use = "tree updater stats should be `report()`ed"] -#[derive(Debug, Clone, Copy, Default)] +#[derive(Clone, Copy, Default)] pub(crate) struct TreeUpdaterStats { pub new_leaves: u64, pub new_internal_nodes: u64, @@ -110,6 +119,24 @@ pub(crate) struct TreeUpdaterStats { pub patch_reads: u64, } +impl fmt::Debug for TreeUpdaterStats { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("TreeUpdaterStats") + .field("new_leaves", &self.new_leaves) + .field("new_internal_nodes", &self.new_internal_nodes) + .field("moved_leaves", &self.moved_leaves) + .field("updated_leaves", &self.updated_leaves) + .field("avg_leaf_level", &self.avg_leaf_level()) + .field("max_leaf_level", &self.max_leaf_level) + .field("key_reads", &self.key_reads) + .field("missing_key_reads", &self.missing_key_reads) + .field("db_reads", &self.db_reads) + .field("patch_reads", &self.patch_reads) + .finish_non_exhaustive() + } +} + impl TreeUpdaterStats { pub(crate) fn update_leaf_levels(&mut self, nibble_count: usize) { let leaf_level = nibble_count as u64 * 4; @@ -118,20 +145,22 @@ impl TreeUpdaterStats { } #[allow(clippy::cast_precision_loss)] // Acceptable for metrics + fn avg_leaf_level(&self) -> f64 { + let touched_leaves = self.new_leaves + self.moved_leaves; + if touched_leaves > 0 { + self.leaf_level_sum as f64 / touched_leaves as f64 + } else { + 0.0 + } + } + pub(crate) fn report(self) { let metrics = &TREE_UPDATE_METRICS; metrics.new_leaves.observe(self.new_leaves); metrics.new_internal_nodes.observe(self.new_internal_nodes); metrics.moved_leaves.observe(self.moved_leaves); metrics.updated_leaves.observe(self.updated_leaves); - - let touched_leaves = self.new_leaves + self.moved_leaves; - let avg_leaf_level = if touched_leaves > 0 { - self.leaf_level_sum as f64 / touched_leaves as f64 - } else { - 0.0 - }; - metrics.avg_leaf_level.observe(avg_leaf_level); + metrics.avg_leaf_level.observe(self.avg_leaf_level()); metrics.max_leaf_level.observe(self.max_leaf_level); if self.key_reads > 0 { @@ -297,7 +326,7 @@ struct PruningMetrics { static PRUNING_METRICS: Global = Global::new(); #[derive(Debug)] -pub(crate) struct PruningStats { +pub struct PruningStats { pub target_retained_version: u64, pub pruned_key_count: usize, pub deleted_stale_key_versions: ops::Range, diff --git a/core/lib/merkle_tree/src/pruning.rs b/core/lib/merkle_tree/src/pruning.rs index 4bbcb8f0bcb5..bf60b8cf956b 100644 --- a/core/lib/merkle_tree/src/pruning.rs +++ b/core/lib/merkle_tree/src/pruning.rs @@ -100,8 +100,9 @@ impl MerkleTreePruner { latest_version.checked_sub(self.past_versions_to_keep) } + #[doc(hidden)] // Used in integration tests; logically private #[allow(clippy::range_plus_one)] // exclusive range is required by `PrunePatchSet` constructor - fn run_once(&mut self) -> Option { + pub fn run_once(&mut self) -> Option { let target_retained_version = self.target_retained_version()?; let min_stale_key_version = self.db.min_stale_key_version()?; let stale_key_new_versions = min_stale_key_version..=target_retained_version; @@ -211,8 +212,10 @@ mod tests { assert!(!stats.has_more_work()); // Check the `PatchSet` implementation of `PruneDatabase`. - assert_eq!(db.roots_mut().len(), 1); - assert!(db.roots_mut().contains_key(&4)); + for version in 0..4 { + assert!(db.root_mut(version).is_none()); + } + assert!(db.root_mut(4).is_some()); } #[test] diff --git a/core/lib/merkle_tree/src/recovery.rs b/core/lib/merkle_tree/src/recovery.rs new file mode 100644 index 000000000000..7e7450596d88 --- /dev/null +++ b/core/lib/merkle_tree/src/recovery.rs @@ -0,0 +1,279 @@ +//! Merkle tree recovery logic. +//! +//! # Overview +//! +//! **Recovery process** is responsible for restoring a Merkle tree from a snapshot. A snapshot +//! consists of all tree entries at a specific tree version. As a result of recovery, we create +//! a Merkle tree with the same entries as the snapshot. Any changes that are applied to the tree +//! afterwards will have the same outcome as if they were applied to the original tree. +//! +//! Importantly, a recovered tree is only *observably* identical to the original tree; it differs +//! in (currently unobservable) node versions. In a recovered tree, all nodes will initially have +//! the same version (the snapshot version), while in the original tree, node versions are distributed +//! from 0 to the snapshot version (both inclusive). +//! +//! Recovery process proceeds as follows: +//! +//! 1. Initialize a tree in the recovery mode. Until recovery is finished, the tree cannot be accessed +//! using ordinary [`MerkleTree`] APIs. +//! 2. Update the tree from a snapshot, which [is fed to the tree](MerkleTreeRecovery::extend()) +//! as [`RecoveryEntry`] chunks. Recovery entries must be ordered by increasing key. +//! 3. Finalize recovery using [`MerkleTreeRecovery::finalize()`]. To check integrity, you may compare +//! [`MerkleTreeRecovery::root_hash()`] to the reference value. +//! +//! The recovery process is tolerant to crashes and may be resumed from the middle. To find the latest +//! recovered key, you may use [`MerkleTreeRecovery::last_processed_key()`]. +//! +//! `RecoveryEntry` chunks are not validated during recovery. They can be authenticated using +//! [`TreeRangeDigest`](crate::TreeRangeDigest)s provided that the tree root hash is authenticated +//! using external means. +//! +//! # Implementation details +//! +//! We require `RecoveryEntry` ordering to simplify tracking the recovery progress. It also makes +//! node updates more efficient. Indeed, it suffices to load a leaf with the greatest key and its ancestors +//! before extending the tree; these nodes are guaranteed to be the *only* DB reads necessary +//! to insert new entries. + +use std::time::Instant; + +use crate::{ + hasher::HashTree, + storage::{PatchSet, PruneDatabase, PrunePatchSet, Storage}, + types::{Key, Manifest, Root, TreeTags, ValueHash}, + MerkleTree, +}; +use zksync_crypto::hasher::blake2::Blake2Hasher; + +/// Entry in a Merkle tree used during recovery. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecoveryEntry { + /// Entry key. + pub key: Key, + /// Entry value. + pub value: ValueHash, + /// Leaf index associated with the entry. It is **not** checked whether leaf indices are well-formed + /// during recovery (e.g., that they are unique). + pub leaf_index: u64, +} + +/// Handle to a Merkle tree during its recovery. +#[derive(Debug)] +pub struct MerkleTreeRecovery<'a, DB> { + db: DB, + hasher: &'a dyn HashTree, + recovered_version: u64, +} + +impl<'a, DB: PruneDatabase> MerkleTreeRecovery<'a, DB> { + /// Creates tree recovery with the default Blake2 hasher. + /// + /// # Panics + /// + /// Panics in the same situations as [`Self::with_hasher()`]. + pub fn new(db: DB, recovered_version: u64) -> Self { + Self::with_hasher(db, recovered_version, &Blake2Hasher) + } + + /// Loads a tree with the specified hasher. + /// + /// # Panics + /// + /// - Panics if the tree DB exists and it's not being recovered, or if it's being recovered + /// for a different tree version. + /// - Panics if the hasher or basic tree parameters (e.g., the tree depth) + /// do not match those of the tree loaded from the database. + pub fn with_hasher(mut db: DB, recovered_version: u64, hasher: &'a dyn HashTree) -> Self { + let manifest = db.manifest(); + let mut manifest = if let Some(manifest) = manifest { + if manifest.version_count > 0 { + let expected_version = manifest.version_count - 1; + assert_eq!( + recovered_version, + expected_version, + "Requested to recover tree version {recovered_version}, but it is currently being recovered \ + for version {expected_version}" + ); + } + manifest + } else { + Manifest { + version_count: recovered_version + 1, + tags: None, + } + }; + + manifest.version_count = recovered_version + 1; + if let Some(tags) = &manifest.tags { + tags.assert_consistency(hasher, true); + } else { + let mut tags = TreeTags::new(hasher); + tags.is_recovering = true; + manifest.tags = Some(tags); + } + db.apply_patch(PatchSet::from_manifest(manifest)); + + Self { + db, + hasher, + recovered_version, + } + } + + /// Returns the root hash of the recovered tree at this point. + pub fn root_hash(&self) -> ValueHash { + let root = self.db.root(self.recovered_version); + let Some(Root::Filled { node, .. }) = root else { + return self.hasher.empty_tree_hash(); + }; + node.hash(&mut self.hasher.into(), 0) + } + + /// Returns the last key processed during the recovery process. + pub fn last_processed_key(&self) -> Option { + let storage = Storage::new(&self.db, self.hasher, self.recovered_version, false); + storage.greatest_key() + } + + /// Extends a tree with a chunk of entries. + /// + /// Entries must be ordered by increasing `key`, and the key of the first entry must be greater + /// than [`Self::last_processed_key()`]. + /// + /// # Panics + /// + /// Panics if entry keys are not correctly ordered. + #[tracing::instrument( + level = "debug", + skip_all, + fields( + recovered_version = self.recovered_version, + entries.len = entries.len(), + %entries.key_range = entries_key_range(&entries), + ), + )] + pub fn extend(&mut self, entries: Vec) { + tracing::debug!("Started extending tree"); + + let started_at = Instant::now(); + let storage = Storage::new(&self.db, self.hasher, self.recovered_version, false); + let patch = storage.extend_during_recovery(entries); + tracing::debug!("Finished processing keys; took {:?}", started_at.elapsed()); + + let started_at = Instant::now(); + self.db.apply_patch(patch); + tracing::debug!("Finished persisting to DB; took {:?}", started_at.elapsed()); + } + + /// Finalizes the recovery process marking it as complete in the tree manifest. + #[tracing::instrument( + level = "debug", + skip_all, + fields(recovered_version = self.recovered_version), + )] + #[allow(clippy::missing_panics_doc, clippy::range_plus_one)] + pub fn finalize(mut self) -> MerkleTree<'a, DB> { + let mut manifest = self.db.manifest().unwrap(); + // ^ `unwrap()` is safe: manifest is inserted into the DB on creation + + let leaf_count = if let Some(root) = self.db.root(self.recovered_version) { + root.leaf_count() + } else { + // Marginal case: an empty tree is recovered (i.e., `extend()` was never called). + let patch = PatchSet::for_empty_root(manifest.clone(), self.recovered_version); + self.db.apply_patch(patch); + 0 + }; + tracing::debug!( + "Finalizing recovery of the Merkle tree with {leaf_count} key–value entries" + ); + + let started_at = Instant::now(); + let stale_keys = self.db.stale_keys(self.recovered_version); + let stale_keys_len = stale_keys.len(); + tracing::debug!("Pruning {stale_keys_len} accumulated stale keys"); + let prune_patch = PrunePatchSet::new( + stale_keys, + self.recovered_version..self.recovered_version + 1, + ); + self.db.prune(prune_patch); + tracing::debug!( + "Pruned {stale_keys_len} stale keys in {:?}", + started_at.elapsed() + ); + + manifest + .tags + .get_or_insert_with(|| TreeTags::new(self.hasher)) + .is_recovering = false; + self.db.apply_patch(PatchSet::from_manifest(manifest)); + tracing::debug!("Updated tree manifest to mark recovery as complete"); + + // We don't need additional integrity checks since they were performed in the constructor + MerkleTree { + db: self.db, + hasher: self.hasher, + } + } +} + +fn entries_key_range(entries: &[RecoveryEntry]) -> String { + let (Some(first), Some(last)) = (entries.first(), entries.last()) else { + return "(empty)".to_owned(); + }; + format!("{:0>64x}..={:0>64x}", first.key, last.key) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{hasher::HasherWithStats, types::LeafNode}; + + #[test] + #[should_panic(expected = "Tree is expected to be in the process of recovery")] + fn recovery_for_initialized_tree() { + let mut db = PatchSet::default(); + MerkleTreeRecovery::new(&mut db, 123).finalize(); + MerkleTreeRecovery::new(db, 123); + } + + #[test] + #[should_panic(expected = "Requested to recover tree version 42")] + fn recovery_for_different_version() { + let mut db = PatchSet::default(); + MerkleTreeRecovery::new(&mut db, 123); + MerkleTreeRecovery::new(&mut db, 42); + } + + #[test] + fn recovering_empty_tree() { + let tree = MerkleTreeRecovery::new(PatchSet::default(), 42).finalize(); + assert_eq!(tree.latest_version(), Some(42)); + assert_eq!(tree.root(42), Some(Root::Empty)); + } + + #[test] + fn recovering_tree_with_single_node() { + let mut recovery = MerkleTreeRecovery::new(PatchSet::default(), 42); + let recovery_entry = RecoveryEntry { + key: Key::from(123), + value: ValueHash::repeat_byte(1), + leaf_index: 1, + }; + recovery.extend(vec![recovery_entry]); + let tree = recovery.finalize(); + + assert_eq!(tree.latest_version(), Some(42)); + let mut hasher = HasherWithStats::from(&Blake2Hasher as &dyn HashTree); + assert_eq!( + tree.latest_root_hash(), + LeafNode::new( + recovery_entry.key, + recovery_entry.value, + recovery_entry.leaf_index + ) + .hash(&mut hasher, 0) + ); + tree.verify_consistency(42).unwrap(); + } +} diff --git a/core/lib/merkle_tree/src/storage/database.rs b/core/lib/merkle_tree/src/storage/database.rs index 0c1e494dc21e..4cefd52d9a21 100644 --- a/core/lib/merkle_tree/src/storage/database.rs +++ b/core/lib/merkle_tree/src/storage/database.rs @@ -109,7 +109,10 @@ impl Database for PatchSet { } fn try_root(&self, version: u64) -> Result, DeserializeError> { - Ok(self.roots.get(&version).cloned()) + let Some(patch) = self.patches_by_version.get(&version) else { + return Ok(None); + }; + Ok(patch.root.clone()) } fn try_tree_node( @@ -117,10 +120,8 @@ impl Database for PatchSet { key: &NodeKey, is_leaf: bool, ) -> Result, DeserializeError> { - let node = self - .nodes_by_version - .get(&key.version) - .and_then(|nodes| nodes.get(key)); + let patch_with_node = self.patches_by_version.get(&key.version); + let node = patch_with_node.and_then(|patch| patch.nodes.get(key)); let Some(node) = node.cloned() else { return Ok(None); }; @@ -134,21 +135,44 @@ impl Database for PatchSet { Ok(Some(node)) } - fn apply_patch(&mut self, other: PatchSet) { + fn apply_patch(&mut self, mut other: PatchSet) { + if let Some(other_updated_version) = other.updated_version { + if let Some(updated_version) = self.updated_version { + assert_eq!( + other_updated_version, updated_version, + "Cannot merge patches with different updated versions" + ); + + let patch = self.patches_by_version.get_mut(&updated_version).unwrap(); + let other_patch = other.patches_by_version.remove(&updated_version).unwrap(); + // ^ `unwrap()`s are safe by design. + patch.merge(other_patch); + } else { + assert!( + self.patches_by_version.keys().all(|&ver| ver > other_updated_version), + "Cannot update {self:?} from {other:?}; this would break the update version invariant \ + (the update version being lesser than all inserted versions)" + ); + self.updated_version = Some(other_updated_version); + } + } + let new_version_count = other.manifest.version_count; if new_version_count < self.manifest.version_count { - // Remove obsolete roots and nodes from the patch. - self.roots.retain(|&version, _| version < new_version_count); - self.nodes_by_version - .retain(|&version, _| version < new_version_count); - self.stale_keys_by_version + // Remove obsolete sub-patches from the patch. + self.patches_by_version .retain(|&version, _| version < new_version_count); } self.manifest = other.manifest; - self.roots.extend(other.roots); - self.nodes_by_version.extend(other.nodes_by_version); - self.stale_keys_by_version - .extend(other.stale_keys_by_version); + self.patches_by_version.extend(other.patches_by_version); + for (version, stale_keys) in other.stale_keys_by_version { + self.stale_keys_by_version + .entry(version) + .or_default() + .extend(stale_keys); + } + // `PatchSet` invariants hold by construction: the updated version (if set) is still lower + // than all other versions by design. } } @@ -170,9 +194,28 @@ impl Patched { } pub(crate) fn patched_versions(&self) -> Vec { - self.patch - .as_ref() - .map_or_else(Vec::new, |patch| patch.roots.keys().copied().collect()) + self.patch.as_ref().map_or_else(Vec::new, |patch| { + patch.patches_by_version.keys().copied().collect() + }) + } + + /// Returns the value from the patch and a flag whether this value is final (i.e., a DB lookup + /// is not required). + fn lookup_patch(&self, key: &NodeKey, is_leaf: bool) -> (Option, bool) { + let Some(patch) = &self.patch else { + return (None, false); + }; + if patch.is_new_version(key.version) { + return (patch.tree_node(key, is_leaf), true); + } + let could_be_in_updated_patch = patch.updated_version == Some(key.version); + if could_be_in_updated_patch { + // Unlike with new versions, we must look both in the update patch and in the original DB. + if let Some(node) = patch.tree_node(key, is_leaf) { + return (Some(node), true); + } + } + (None, false) } /// Provides readonly access to the wrapped DB. @@ -223,8 +266,9 @@ impl Database for Patched { fn try_root(&self, version: u64) -> Result, DeserializeError> { if let Some(patch) = &self.patch { - if patch.is_responsible_for_version(version) { - return Ok(patch.roots.get(&version).cloned()); + let has_root = patch.is_new_version(version) || patch.updated_version == Some(version); + if has_root { + return patch.try_root(version); } } self.inner.try_root(version) @@ -235,32 +279,41 @@ impl Database for Patched { key: &NodeKey, is_leaf: bool, ) -> Result, DeserializeError> { - let Some(patch) = &self.patch else { - return self.inner.try_tree_node(key, is_leaf); - }; - - if patch.is_responsible_for_version(key.version) { - patch.try_tree_node(key, is_leaf) // take use of debug assertions + let (patch_node, is_final) = self.lookup_patch(key, is_leaf); + if is_final { + Ok(patch_node) + } else if let Some(node) = patch_node { + Ok(Some(node)) } else { self.inner.try_tree_node(key, is_leaf) } } fn tree_nodes(&self, keys: &NodeKeys) -> Vec> { - let Some(patch) = &self.patch else { + if self.patch.is_none() { return self.inner.tree_nodes(keys); - }; + } - let mut is_in_patch = Vec::with_capacity(keys.len()); - let (patch_keys, db_keys): (Vec<_>, Vec<_>) = keys.iter().partition(|(key, _)| { - let flag = patch.is_responsible_for_version(key.version); - is_in_patch.push(flag); - flag - }); + let mut is_in_patch = vec![false; keys.len()]; + let mut patch_values = vec![]; + for (i, (key, is_leaf)) in keys.iter().enumerate() { + let (patch_node, is_final) = self.lookup_patch(key, *is_leaf); + if is_final { + patch_values.push(patch_node); + is_in_patch[i] = true; + } else if let Some(node) = patch_node { + patch_values.push(Some(node)); + is_in_patch[i] = true; + } + } + let db_keys: Vec<_> = keys + .iter() + .zip(&is_in_patch) + .filter_map(|(&key, &is_in_patch)| (!is_in_patch).then_some(key)) + .collect(); - let mut patch_values = patch.tree_nodes(&patch_keys).into_iter(); + let mut patch_values = patch_values.into_iter(); let mut db_values = self.inner.tree_nodes(&db_keys).into_iter(); - let values = is_in_patch.into_iter().map(|is_in_patch| { if is_in_patch { patch_values.next().unwrap() @@ -346,10 +399,13 @@ impl PruneDatabase for PatchSet { fn prune(&mut self, patch: PrunePatchSet) { for key in &patch.pruned_node_keys { + let Some(patch) = self.patches_by_version.get_mut(&key.version) else { + continue; + }; if key.is_empty() { - self.roots.remove(&key.version); - } else if let Some(nodes) = self.nodes_by_version.get_mut(&key.version) { - nodes.remove(key); + patch.root = None; + } else { + patch.nodes.remove(key); } } @@ -364,10 +420,97 @@ mod tests { use super::*; use crate::{ - storage::tests::{create_patch, generate_nodes, FIRST_KEY}, + storage::{ + tests::{create_patch, generate_nodes, FIRST_KEY}, + Operation, + }, types::{InternalNode, Nibbles}, }; + #[test] + fn patch_set_with_update() { + let manifest = Manifest::new(10, &()); + let old_root = Root::new(2, Node::Internal(InternalNode::default())); + let nodes = generate_nodes(9, &[1, 2]); + let mut patch = PatchSet::new( + manifest, + 9, + old_root.clone(), + nodes.clone(), + vec![], + Operation::Update, + ); + + for ver in (0..9).chain(10..20) { + assert!(patch.root(ver).is_none()); + } + assert_eq!(patch.root(9).unwrap(), old_root); + let (&node_key, expected_node) = nodes.iter().next().unwrap(); + let node = patch.tree_node(&node_key, true).unwrap(); + assert_eq!(node, *expected_node); + + let new_nodes = generate_nodes(10, &[3, 4]); + let manifest = Manifest::new(11, &()); + let new_root = Root::new(4, Node::Internal(InternalNode::default())); + let new_patch = PatchSet::new( + manifest, + 10, + new_root.clone(), + new_nodes.clone(), + vec![], + Operation::Insert, + ); + patch.apply_patch(new_patch); + + for ver in (0..9).chain(11..20) { + assert!(patch.root(ver).is_none()); + } + assert_eq!(patch.root(9).unwrap(), old_root); + assert_eq!(patch.root(10).unwrap(), new_root); + let (&node_key, expected_node) = nodes.iter().next().unwrap(); + let node = patch.tree_node(&node_key, true).unwrap(); + assert_eq!(node, *expected_node); + let (&node_key, expected_node) = new_nodes.iter().next().unwrap(); + let node = patch.tree_node(&node_key, true).unwrap(); + assert_eq!(node, *expected_node); + } + + #[test] + fn merging_two_update_patches() { + let manifest = Manifest::new(10, &()); + let old_root = Root::new(2, Node::Internal(InternalNode::default())); + let nodes = generate_nodes(9, &[1, 2]); + let mut patch = PatchSet::new( + manifest.clone(), + 9, + old_root, + nodes.clone(), + vec![], + Operation::Update, + ); + + let new_nodes = generate_nodes(9, &[3, 4]); + let new_root = Root::new(4, Node::Internal(InternalNode::default())); + let new_patch = PatchSet::new( + manifest, + 9, + new_root.clone(), + new_nodes.clone(), + vec![], + Operation::Update, + ); + patch.apply_patch(new_patch); + + for ver in (0..9).chain(10..20) { + assert!(patch.root(ver).is_none()); + } + assert_eq!(patch.root(9).unwrap(), new_root); + for (&node_key, expected_node) in nodes.iter().chain(&new_nodes) { + let node = patch.tree_node(&node_key, true).unwrap(); + assert_eq!(node, *expected_node); + } + } + #[test] fn requesting_nodes_in_patched_db() { let root = Root::new(2, Node::Internal(InternalNode::default())); @@ -431,4 +574,55 @@ mod tests { [Some(_), Some(_), None, None, None, None, Some(_), Some(_), Some(_)] ); } + + #[test] + fn patched_db_with_update_patch() { + let manifest = Manifest::new(10, &()); + let old_root = Root::new(2, Node::Internal(InternalNode::default())); + let nodes = generate_nodes(9, &[1, 2]); + let db = PatchSet::new( + manifest.clone(), + 9, + old_root.clone(), + nodes.clone(), + vec![], + Operation::Update, + ); + let mut patched = Patched::new(db); + + let new_nodes = generate_nodes(9, &[3, 4]); + let new_root = Root::new(4, Node::Internal(InternalNode::default())); + let new_patch = PatchSet::new( + manifest, + 9, + new_root.clone(), + new_nodes.clone(), + vec![], + Operation::Update, + ); + patched.apply_patch(new_patch); + + for ver in (0..9).chain(10..20) { + assert!(patched.root(ver).is_none()); + } + assert_eq!(patched.root(9).unwrap(), new_root); + for (&node_key, expected_node) in nodes.iter().chain(&new_nodes) { + let node = patched.tree_node(&node_key, true).unwrap(); + assert_eq!(node, *expected_node); + } + + let requested_keys: Vec<_> = nodes + .keys() + .chain(new_nodes.keys()) + .map(|&key| (key, true)) + .collect(); + let retrieved_nodes = patched.tree_nodes(&requested_keys); + assert_eq!(retrieved_nodes.len(), requested_keys.len()); + for ((key, _), node) in requested_keys.iter().zip(retrieved_nodes) { + assert_eq!( + node.unwrap(), + *nodes.get(key).unwrap_or_else(|| &new_nodes[key]) + ); + } + } } diff --git a/core/lib/merkle_tree/src/storage/mod.rs b/core/lib/merkle_tree/src/storage/mod.rs index a7553727467a..baea778cf93f 100644 --- a/core/lib/merkle_tree/src/storage/mod.rs +++ b/core/lib/merkle_tree/src/storage/mod.rs @@ -18,6 +18,7 @@ pub use self::{ use crate::{ hasher::HashTree, metrics::{TreeUpdaterStats, BLOCK_TIMINGS, GENERAL_METRICS}, + recovery::RecoveryEntry, types::{ BlockOutput, ChildRef, InternalNode, Key, LeafNode, Manifest, Nibbles, Node, Root, TreeLogEntry, TreeTags, ValueHash, @@ -25,6 +26,14 @@ use crate::{ utils::increment_counter, }; +/// Tree operation: either inserting a new version or updating an existing one (the latter is only +/// used during tree recovery). +#[derive(Debug, Clone, Copy)] +enum Operation { + Insert, + Update, +} + /// Mutable storage encapsulating AR16MT update logic. #[derive(Debug)] struct TreeUpdater { @@ -92,6 +101,14 @@ impl TreeUpdater { longest_prefixes } + /// Loads the greatest key from the database. + fn load_greatest_key(&mut self, db: &DB) -> Option<(LeafNode, Nibbles)> { + let (leaf, load_result) = self.patch_set.load_greatest_key(db)?; + self.metrics.db_reads += load_result.db_reads; + assert_eq!(load_result.longest_prefixes.len(), 1); + Some((leaf, load_result.longest_prefixes[0])) + } + /// Inserts or updates a value hash for the specified `key`. This implementation /// is almost verbatim the algorithm described in the Jellyfish Merkle tree white paper. /// The algorithm from the paper is as follows: @@ -120,7 +137,7 @@ impl TreeUpdater { parent_nibbles: &Nibbles, leaf_index_fn: impl FnOnce() -> u64, ) -> (TreeLogEntry, NewLeafData) { - let version = self.patch_set.version(); + let version = self.patch_set.root_version(); let traverse_outcome = self.patch_set.traverse(key, parent_nibbles); let (log, leaf_data) = match traverse_outcome { TraverseOutcome::LeafMatch(nibbles, mut leaf) => { @@ -132,12 +149,7 @@ impl TreeUpdater { } TraverseOutcome::LeafMismatch(nibbles, leaf) => { - if let Some((parent_nibbles, last_nibble)) = nibbles.split_last() { - self.patch_set - .child_ref_mut(&parent_nibbles, last_nibble) - .unwrap() - .is_leaf = false; - } + self.update_moved_leaf_ref(&nibbles); let mut nibble_idx = nibbles.nibble_count(); loop { @@ -203,15 +215,26 @@ impl TreeUpdater { // Traverse nodes up to the root level and update `ChildRef.version`. let mut cursor = traverse_outcome.position(); while let Some((parent_nibbles, last_nibble)) = cursor.split_last() { - self.patch_set + let child_ref = self + .patch_set .child_ref_mut(&parent_nibbles, last_nibble) - .unwrap() - .version = version; + .unwrap(); + child_ref.version = child_ref.version.max(version); cursor = parent_nibbles; } (log, leaf_data) } + + fn update_moved_leaf_ref(&mut self, leaf_nibbles: &Nibbles) { + if let Some((parent_nibbles, last_nibble)) = leaf_nibbles.split_last() { + let child_ref = self + .patch_set + .child_ref_mut(&parent_nibbles, last_nibble) + .unwrap(); + child_ref.is_leaf = false; + } + } } /// [`TreeUpdater`] together with a link to the database. @@ -221,22 +244,33 @@ pub(crate) struct Storage<'a, DB: ?Sized> { hasher: &'a dyn HashTree, manifest: Manifest, leaf_count: u64, + operation: Operation, updater: TreeUpdater, } impl<'a, DB: Database + ?Sized> Storage<'a, DB> { /// Creates storage for a new version of the tree. - pub fn new(db: &'a DB, hasher: &'a dyn HashTree, version: u64) -> Self { + pub fn new( + db: &'a DB, + hasher: &'a dyn HashTree, + version: u64, + create_new_version: bool, + ) -> Self { let mut manifest = db.manifest().unwrap_or_default(); if manifest.tags.is_none() { manifest.tags = Some(TreeTags::new(hasher)); } manifest.version_count = version + 1; - let root = if version == 0 { - Root::Empty + let base_version = if create_new_version { + version.checked_sub(1) + } else { + Some(version) + }; + let root = if let Some(base_version) = base_version { + db.root(base_version).unwrap_or(Root::Empty) } else { - db.root(version - 1).expect("no previous root") + Root::Empty }; Self { @@ -244,6 +278,11 @@ impl<'a, DB: Database + ?Sized> Storage<'a, DB> { hasher, manifest, leaf_count: root.leaf_count(), + operation: if create_new_version { + Operation::Insert + } else { + Operation::Update + }, updater: TreeUpdater::new(version, root), } } @@ -254,7 +293,8 @@ impl<'a, DB: Database + ?Sized> Storage<'a, DB> { let load_nodes_latency = BLOCK_TIMINGS.load_nodes.start(); let sorted_keys = SortedKeys::new(key_value_pairs.iter().map(|(key, _)| *key)); let parent_nibbles = self.updater.load_ancestors(&sorted_keys, self.db); - load_nodes_latency.observe(); + let load_nodes_latency = load_nodes_latency.observe(); + tracing::debug!("Load stage took {load_nodes_latency:?}"); let extend_patch_latency = BLOCK_TIMINGS.extend_patch.start(); let mut logs = Vec::with_capacity(key_value_pairs.len()); @@ -264,7 +304,8 @@ impl<'a, DB: Database + ?Sized> Storage<'a, DB> { }); logs.push(log); } - extend_patch_latency.observe(); + let extend_patch_latency = extend_patch_latency.observe(); + tracing::debug!("Tree traversal stage took {extend_patch_latency:?}"); let leaf_count = self.leaf_count; let (root_hash, patch) = self.finalize(); @@ -276,16 +317,64 @@ impl<'a, DB: Database + ?Sized> Storage<'a, DB> { (output, patch) } + pub fn greatest_key(mut self) -> Option { + Some(self.updater.load_greatest_key(self.db)?.0.full_key) + } + + pub fn extend_during_recovery(mut self, recovery_entries: Vec) -> PatchSet { + let (mut prev_key, mut prev_nibbles) = match self.updater.load_greatest_key(self.db) { + Some((leaf, nibbles)) => (Some(leaf.full_key), nibbles), + None => (None, Nibbles::EMPTY), + }; + + let extend_patch_latency = BLOCK_TIMINGS.extend_patch.start(); + for entry in recovery_entries { + if let Some(prev_key) = prev_key { + assert!( + entry.key > prev_key, + "Recovery entries must be ordered by increasing key (previous key: {prev_key:0>64x}, \ + offending entry: {entry:?})" + ); + } + prev_key = Some(entry.key); + + let key_nibbles = Nibbles::new(&entry.key, prev_nibbles.nibble_count()); + let parent_nibbles = prev_nibbles.common_prefix(&key_nibbles); + let (_, new_leaf) = + self.updater + .insert(entry.key, entry.value, &parent_nibbles, || entry.leaf_index); + prev_nibbles = new_leaf.nibbles; + self.leaf_count += 1; + } + let extend_patch_latency = extend_patch_latency.observe(); + tracing::debug!("Tree traversal stage took {extend_patch_latency:?}"); + + let (_, patch) = self.finalize(); + patch + } + fn finalize(self) -> (ValueHash, PatchSet) { + tracing::debug!( + "Finished updating tree; total leaf count: {}, stats: {:?}", + self.leaf_count, + self.updater.metrics + ); self.updater.metrics.report(); - let finalize_patch = BLOCK_TIMINGS.finalize_patch.start(); - let (root_hash, patch, stats) = - self.updater - .patch_set - .finalize(self.manifest, self.leaf_count, self.hasher); + let finalize_patch_latency = BLOCK_TIMINGS.finalize_patch.start(); + let (root_hash, patch, stats) = self.updater.patch_set.finalize( + self.manifest, + self.leaf_count, + self.operation, + self.hasher, + ); GENERAL_METRICS.leaf_count.set(self.leaf_count); - finalize_patch.observe(); + let finalize_patch_latency = finalize_patch_latency.observe(); + tracing::debug!( + "Tree finalization stage took {finalize_patch_latency:?}; hashed {:?}B in {:?}", + stats.hashed_bytes, + stats.hashing_duration + ); stats.report(); (root_hash, patch) diff --git a/core/lib/merkle_tree/src/storage/patch.rs b/core/lib/merkle_tree/src/storage/patch.rs index 1ba52fab5387..9e251bf01782 100644 --- a/core/lib/merkle_tree/src/storage/patch.rs +++ b/core/lib/merkle_tree/src/storage/patch.rs @@ -2,12 +2,16 @@ use rayon::prelude::*; -use std::collections::{hash_map::Entry, HashMap}; +use std::{ + collections::{hash_map::Entry, HashMap}, + iter, + time::Instant, +}; use crate::{ hasher::{HashTree, HasherWithStats, MerklePath}, metrics::HashingStats, - storage::{proofs::SUBTREE_COUNT, SortedKeys, TraverseOutcome}, + storage::{proofs::SUBTREE_COUNT, Operation, SortedKeys, TraverseOutcome}, types::{ ChildRef, InternalNode, Key, LeafNode, Manifest, Nibbles, NibblesBytes, Node, NodeKey, Root, ValueHash, KEY_SIZE, @@ -15,16 +19,32 @@ use crate::{ utils, Database, }; +/// Subset of a [`PatchSet`] corresponding to a specific version. All nodes in the subset +/// have the same version. +#[derive(Debug)] +pub(super) struct PartialPatchSet { + pub root: Option, + // TODO (BFT-130): investigate most efficient ways to store key-value pairs: + // - `HashMap`s indexed by version + // - Full upper levels (i.e., `Vec>`) + pub nodes: HashMap, +} + +impl PartialPatchSet { + pub fn merge(&mut self, other: Self) { + self.root = other.root; + self.nodes.extend(other.nodes); + } +} + /// Raw set of database changes. #[derive(Debug, Default)] -#[cfg_attr(test, derive(Clone))] // Used in tree consistency tests pub struct PatchSet { pub(super) manifest: Manifest, - pub(super) roots: HashMap, - // TODO (BFT-130): investigate most efficient ways to store key-value pairs: - // - `HashMap`s indexed by version - // - Full upper levels (i.e., `Vec>`) - pub(super) nodes_by_version: HashMap>, + pub(super) patches_by_version: HashMap, + /// INVARIANT: If present, `patches_by_version` contains the corresponding version, and it + /// is smaller than all other keys in `patches_by_version`. + pub(super) updated_version: Option, pub(super) stale_keys_by_version: HashMap>, } @@ -32,19 +52,26 @@ impl PatchSet { pub(crate) fn from_manifest(manifest: Manifest) -> Self { Self { manifest, - roots: HashMap::new(), - nodes_by_version: HashMap::new(), + patches_by_version: HashMap::new(), + updated_version: None, stale_keys_by_version: HashMap::new(), } } - pub(super) fn for_empty_root(manifest: Manifest, version: u64) -> Self { + pub(crate) fn for_empty_root(manifest: Manifest, version: u64) -> Self { let stale_keys = if let Some(prev_version) = version.checked_sub(1) { vec![Nibbles::EMPTY.with_version(prev_version)] } else { vec![] }; - Self::new(manifest, version, Root::Empty, HashMap::new(), stale_keys) + Self::new( + manifest, + version, + Root::Empty, + HashMap::new(), + stale_keys, + Operation::Insert, + ) } pub(super) fn new( @@ -53,29 +80,40 @@ impl PatchSet { root: Root, mut nodes: HashMap, mut stale_keys: Vec, + operation: Operation, ) -> Self { debug_assert_eq!(manifest.version_count, version + 1); + debug_assert!(nodes.keys().all(|key| key.version == version)); nodes.shrink_to_fit(); // We never insert into `nodes` later stale_keys.shrink_to_fit(); + let partial_patch = PartialPatchSet { + root: Some(root), + nodes, + }; + let updated_version = match &operation { + Operation::Insert => None, + Operation::Update => Some(version), + }; + Self { manifest, - roots: HashMap::from_iter([(version, root)]), - nodes_by_version: HashMap::from_iter([(version, nodes)]), - stale_keys_by_version: HashMap::from_iter([(version, stale_keys)]), + patches_by_version: HashMap::from([(version, partial_patch)]), + updated_version, + stale_keys_by_version: HashMap::from([(version, stale_keys)]), } } - pub(super) fn is_responsible_for_version(&self, version: u64) -> bool { + pub(super) fn is_new_version(&self, version: u64) -> bool { version >= self.manifest.version_count // this patch truncates `version` - || self.roots.contains_key(&version) + || (self.updated_version != Some(version) && self.patches_by_version.contains_key(&version)) } /// Calculates the number of hashes in `ChildRef`s copied from the previous versions /// of the tree. This allows to estimate redundancy of this `PatchSet`. pub(super) fn copied_hashes_count(&self) -> u64 { - let copied_hashes = self.nodes_by_version.iter().map(|(&version, nodes)| { - let copied_hashes = nodes.values().map(|node| { + let copied_hashes = self.patches_by_version.iter().map(|(&version, patch)| { + let copied_hashes = patch.nodes.values().map(|node| { let Node::Internal(node) = node else { return 0; }; @@ -94,17 +132,25 @@ impl PatchSet { &mut self.manifest } - pub(crate) fn roots_mut(&mut self) -> &mut HashMap { - &mut self.roots + pub(crate) fn root_mut(&mut self, version: u64) -> Option<&mut Root> { + let patch = self.patches_by_version.get_mut(&version)?; + patch.root.as_mut() + } + + pub(crate) fn remove_root(&mut self, version: u64) { + let patch = self.patches_by_version.get_mut(&version).unwrap(); + patch.root = None; } pub(crate) fn nodes_mut(&mut self) -> impl Iterator + '_ { - self.nodes_by_version.values_mut().flatten() + self.patches_by_version + .values_mut() + .flat_map(|patch| &mut patch.nodes) } pub(crate) fn remove_node(&mut self, key: &NodeKey) { - let nodes = self.nodes_by_version.get_mut(&key.version).unwrap(); - nodes.remove(key); + let patch = self.patches_by_version.get_mut(&key.version).unwrap(); + patch.nodes.remove(key); } } @@ -114,23 +160,13 @@ impl PatchSet { struct WorkingNode { inner: Node, prev_version: Option, - is_changed: bool, } impl WorkingNode { - fn unchanged(inner: Node, prev_version: u64) -> Self { - Self { - inner, - prev_version: Some(prev_version), - is_changed: false, - } - } - - fn changed(inner: Node, prev_version: Option) -> Self { + fn new(inner: Node, prev_version: Option) -> Self { Self { inner, prev_version, - is_changed: true, } } } @@ -148,7 +184,7 @@ pub(crate) struct LoadAncestorsResult { /// a Merkle tree. #[derive(Debug)] pub(crate) struct WorkingPatchSet { - version: u64, + root_version: u64, // Group changes by `nibble_count` (which is linearly tied to the tree depth: // `depth == nibble_count * 4`) so that we can compute hashes for all changed nodes // in a single traversal in `Self::finalize()`. @@ -156,23 +192,23 @@ pub(crate) struct WorkingPatchSet { } impl WorkingPatchSet { - pub fn new(version: u64, root: Root) -> Self { + pub fn new(root_version: u64, root: Root) -> Self { let changes_by_nibble_count = match root { Root::Filled { node, .. } => { - let root_node = WorkingNode::changed(node, version.checked_sub(1)); + let root_node = WorkingNode::new(node, root_version.checked_sub(1)); let root_level = [(*Nibbles::EMPTY.bytes(), root_node)]; vec![HashMap::from_iter(root_level)] } Root::Empty => Vec::new(), }; Self { - version, + root_version, changes_by_nibble_count, } } - pub fn version(&self) -> u64 { - self.version + pub fn root_version(&self) -> u64 { + self.root_version } pub fn get(&self, nibbles: &Nibbles) -> Option<&Node> { @@ -190,30 +226,20 @@ impl WorkingPatchSet { } let level = &mut self.changes_by_nibble_count[key.nibble_count()]; - // We use `Entry` API to ensure that `prev_version`is correctly retained + // We use `Entry` API to ensure that `prev_version` is correctly retained // in existing `WorkingNode`s. match level.entry(*key.bytes()) { Entry::Vacant(entry) => { - entry.insert(WorkingNode::changed(node, None)); + entry.insert(WorkingNode::new(node, None)); } Entry::Occupied(mut entry) => { entry.get_mut().inner = node; - entry.get_mut().is_changed = true; } } } /// Marks the retrieved node as changed. pub fn get_mut(&mut self, key: &Nibbles) -> Option<&mut Node> { - let level = self.changes_by_nibble_count.get_mut(key.nibble_count())?; - let node = level.get_mut(key.bytes())?; - node.is_changed = true; - Some(&mut node.inner) - } - - /// Analogue of [`Self::get_mut()`] that doesn't mark the node as changed. - /// This should only be used if the only updated part of the node is its cache. - pub fn get_mut_without_updating(&mut self, key: &Nibbles) -> Option<&mut Node> { let level = self.changes_by_nibble_count.get_mut(key.nibble_count())?; let node = level.get_mut(key.bytes())?; Some(&mut node.inner) @@ -235,10 +261,10 @@ impl WorkingPatchSet { /// The pushed nodes are not marked as changed, so this method should only be used /// if the nodes are loaded from DB. - pub fn push_level_from_db<'a>(&mut self, level: impl Iterator) { + fn push_level_from_db<'a>(&mut self, level: impl Iterator) { let level = level .map(|(key, node)| { - let node = WorkingNode::unchanged(node, key.version); + let node = WorkingNode::new(node, Some(key.version)); (*key.nibbles.bytes(), node) }) .collect(); @@ -254,7 +280,7 @@ impl WorkingPatchSet { let leaf = *leaf; let first_nibble = Nibbles::nibble(&leaf.full_key, 0); let mut internal_node = InternalNode::default(); - internal_node.insert_child_ref(first_nibble, ChildRef::leaf(self.version)); + internal_node.insert_child_ref(first_nibble, ChildRef::leaf(self.root_version)); self.insert(Nibbles::EMPTY, internal_node.clone().into()); self.insert(Nibbles::new(&leaf.full_key, 1), leaf.into()); internal_node @@ -270,7 +296,7 @@ impl WorkingPatchSet { /// Splits this patch set by the first nibble of the contained keys. pub fn split(self) -> [Self; SUBTREE_COUNT] { let mut parts = [(); SUBTREE_COUNT].map(|()| Self { - version: self.version, + root_version: self.root_version, changes_by_nibble_count: vec![HashMap::new(); self.changes_by_nibble_count.len()], }); @@ -293,7 +319,7 @@ impl WorkingPatchSet { } pub fn merge(&mut self, other: Self) { - debug_assert_eq!(self.version, other.version); + debug_assert_eq!(self.root_version, other.root_version); let other_len = other.changes_by_nibble_count.len(); if self.changes_by_nibble_count.len() < other_len { @@ -318,85 +344,118 @@ impl WorkingPatchSet { } } - fn remove_unchanged_nodes(&mut self) { - // Do not remove the root node in any case since it has special role in finalization. - for level in self.changes_by_nibble_count.iter_mut().skip(1) { - level.retain(|_, node| node.is_changed); - } - } - - fn stale_keys(&self) -> Vec { - let levels = self.changes_by_nibble_count.iter().enumerate(); - let stale_keys = levels.flat_map(|(nibble_count, level)| { - level.iter().filter_map(move |(nibbles, node)| { - node.prev_version.map(|prev_version| { - let nibbles = Nibbles::from_parts(*nibbles, nibble_count); - nibbles.with_version(prev_version) - }) - }) - }); - stale_keys.collect() - } - /// Computes hashes and serializes this changeset. - pub fn finalize( - mut self, + pub(super) fn finalize( + self, manifest: Manifest, leaf_count: u64, + operation: Operation, hasher: &dyn HashTree, ) -> (ValueHash, PatchSet, HashingStats) { - self.remove_unchanged_nodes(); - let stale_keys = self.stale_keys(); - let metrics = HashingStats::default(); - + let mut stats = HashingStats::default(); + let (root_hash, patch) = self.finalize_inner( + manifest, + leaf_count, + operation, + |nibble_count, level_changes| { + let started_at = Instant::now(); + let tree_level = nibble_count * 4; + // `into_par_iter()` below uses `rayon` to parallelize hash computations. + let output = level_changes + .into_par_iter() + .map_init( + || hasher.with_stats(&stats), + |hasher, (nibbles, node)| { + let nibbles = Nibbles::from_parts(nibbles, nibble_count); + (nibbles, Some(node.inner.hash(hasher, tree_level)), node) + }, + ) + .collect::>(); + stats.hashing_duration += started_at.elapsed(); + output + }, + ); + let root_hash = root_hash.unwrap_or_else(|| hasher.empty_tree_hash()); + (root_hash, patch, stats) + } + + fn finalize_inner( + self, + manifest: Manifest, + leaf_count: u64, + operation: Operation, + mut map_level_changes: impl FnMut(usize, HashMap) -> I, + ) -> (Option, PatchSet) + where + I: IntoIterator, WorkingNode)>, + { let mut changes_by_nibble_count = self.changes_by_nibble_count; - if changes_by_nibble_count.is_empty() { + let len = changes_by_nibble_count.iter().map(HashMap::len).sum(); + if len == 0 { // The tree is empty and there is no root present. - let patch = PatchSet::for_empty_root(manifest, self.version); - return (hasher.empty_tree_hash(), patch, metrics); + return (None, PatchSet::for_empty_root(manifest, self.root_version)); } - let len = changes_by_nibble_count.iter().map(HashMap::len).sum(); let mut patched_nodes = HashMap::with_capacity(len); + let mut stale_keys = vec![]; // Compute hashes for the changed nodes with decreasing nibble count (i.e., topologically // sorted) and store the computed hash in the parent nodes. while let Some(level_changes) = changes_by_nibble_count.pop() { let nibble_count = changes_by_nibble_count.len(); - let tree_level = nibble_count * 4; - // `into_par_iter()` below uses `rayon` to parallelize hash computations. - let hashed_nodes: Vec<_> = level_changes - .into_par_iter() - .map_init( - || hasher.with_stats(&metrics), - |hasher, (nibbles, node)| { - let nibbles = Nibbles::from_parts(nibbles, nibble_count); - (nibbles, node.inner.hash(hasher, tree_level), node) - }, - ) - .collect(); + let hashed_nodes = map_level_changes(nibble_count, level_changes); for (nibbles, node_hash, node) in hashed_nodes { - if let Some(upper_level_changes) = changes_by_nibble_count.last_mut() { - let (parent_nibbles, last_nibble) = nibbles.split_last().unwrap(); - let parent = upper_level_changes.get_mut(parent_nibbles.bytes()).unwrap(); - let Node::Internal(parent) = &mut parent.inner else { - unreachable!("Node parent must be an internal node"); + let node_version = + if let Some(upper_level_changes) = changes_by_nibble_count.last_mut() { + let (parent_nibbles, last_nibble) = nibbles.split_last().unwrap(); + let parent = upper_level_changes.get_mut(parent_nibbles.bytes()).unwrap(); + let Node::Internal(parent) = &mut parent.inner else { + unreachable!("Node parent must be an internal node"); + }; + // ^ `unwrap()`s are safe by construction: the parent of any changed node + // is an `InternalNode` that must be in the change set as well. + let self_ref = parent.child_ref_mut(last_nibble).unwrap(); + // ^ `unwrap()` is safe by construction: the parent node must reference + // the currently considered child. + if let Some(node_hash) = node_hash { + self_ref.hash = node_hash; + } + self_ref.version + } else { + // We're at the root node level. + if matches!(operation, Operation::Insert) { + // The root node is always replaced for inserts and is never replaced for updated. + if let Some(prev_version) = node.prev_version { + stale_keys.push(nibbles.with_version(prev_version)); + } + } + + let root = Root::new(leaf_count, node.inner); + let patch = PatchSet::new( + manifest, + self.root_version, + root, + patched_nodes, + stale_keys, + operation, + ); + return (node_hash, patch); }; - // ^ `unwrap()`s are safe by construction: the parent of any changed node - // is an `InternalNode` that must be in the change set as well. - let self_ref = parent.child_ref_mut(last_nibble).unwrap(); - // ^ `unwrap()` is safe by construction: the parent node must reference - // the currently considered child. - self_ref.hash = node_hash; - } else { - // We're at the root node level. - let root = Root::new(leaf_count, node.inner); - let patch = - PatchSet::new(manifest, self.version, root, patched_nodes, stale_keys); - return (node_hash, patch, metrics); - } - patched_nodes.insert(nibbles.with_version(self.version), node.inner); + let was_replaced = node + .prev_version + .map_or(true, |prev_version| prev_version < node_version); + if was_replaced { + if let Some(prev_version) = node.prev_version { + stale_keys.push(nibbles.with_version(prev_version)); + } + } + if was_replaced || matches!(operation, Operation::Update) { + // All nodes in the patch set are updated for the update operation, regardless + // of the version change. For insert operations, we only should update nodes + // with the changed version. + patched_nodes.insert(nibbles.with_version(node_version), node.inner); + } } } unreachable!("We should have returned when the root node was encountered above"); @@ -408,24 +467,19 @@ impl WorkingPatchSet { Some(node.inner) } - pub fn finalize_without_hashing(mut self, manifest: Manifest, leaf_count: u64) -> PatchSet { - self.remove_unchanged_nodes(); - let stale_keys = self.stale_keys(); - - let Some(root) = self.take_root() else { - return PatchSet::for_empty_root(manifest, self.version); - }; - let root = Root::new(leaf_count, root); - - let levels = self.changes_by_nibble_count.drain(1..); - let nodes = levels.enumerate().flat_map(|(i, level)| { - let nibble_count = i + 1; - level.into_iter().map(move |(nibbles, node)| { - let nibbles = Nibbles::from_parts(nibbles, nibble_count); - (nibbles.with_version(self.version), node.inner) - }) - }); - PatchSet::new(manifest, self.version, root, nodes.collect(), stale_keys) + pub fn finalize_without_hashing(self, manifest: Manifest, leaf_count: u64) -> PatchSet { + let (_, patch) = self.finalize_inner( + manifest, + leaf_count, + Operation::Insert, + |nibble_count, level_changes| { + level_changes.into_iter().map(move |(nibbles, node)| { + let nibbles = Nibbles::from_parts(nibbles, nibble_count); + (nibbles, None, node) + }) + }, + ); + patch } /// Loads ancestor nodes for all keys in `sorted_keys`. @@ -530,6 +584,36 @@ impl WorkingPatchSet { unreachable!("We must have encountered a leaf or missing node when traversing"); } + pub fn load_greatest_key( + &mut self, + db: &DB, + ) -> Option<(LeafNode, LoadAncestorsResult)> { + let mut nibbles = Nibbles::EMPTY; + let mut db_reads = 0; + let greatest_leaf = loop { + match self.get(&nibbles) { + None => return None, + Some(Node::Leaf(leaf)) => break *leaf, + Some(Node::Internal(node)) => { + let (next_nibble, child_ref) = node.last_child_ref(); + nibbles = nibbles.push(next_nibble).unwrap(); + // ^ `unwrap()` is safe; there can be no internal nodes on the bottommost tree level + let child_key = nibbles.with_version(child_ref.version); + let child_node = db.tree_node(&child_key, child_ref.is_leaf).unwrap(); + // ^ `unwrap()` is safe by construction + self.push_level_from_db(iter::once((&child_key, child_node))); + db_reads += 1; + } + } + }; + + let result = LoadAncestorsResult { + longest_prefixes: vec![nibbles], + db_reads, + }; + Some((greatest_leaf, result)) + } + /// Creates a Merkle proof for the specified `key`, which has given `parent_nibbles` /// in this patch set. `root_nibble_count` specifies to which level the proof needs to be constructed. pub(crate) fn create_proof( @@ -572,7 +656,7 @@ impl WorkingPatchSet { break; } - let parent = self.get_mut_without_updating(&parent_nibbles); + let parent = self.get_mut(&parent_nibbles); let Some(Node::Internal(parent)) = parent else { unreachable!() }; @@ -594,7 +678,10 @@ impl WorkingPatchSet { #[cfg(test)] mod tests { use super::*; - use crate::types::{Key, LeafNode}; + use crate::{ + storage::Storage, + types::{Key, LeafNode}, + }; fn patch_len(patch: &WorkingPatchSet) -> usize { patch.changes_by_nibble_count.iter().map(HashMap::len).sum() @@ -644,4 +731,50 @@ mod tests { } assert_eq!(patch_len(&merged), all_nibbles.len() + 1); } + + #[test] + fn loading_greatest_key() { + // Test empty DB. + let mut patch = WorkingPatchSet::new(0, Root::Empty); + let load_result = patch.load_greatest_key(&PatchSet::default()); + assert!(load_result.is_none()); + + // Test DB with a single entry. + let mut db = PatchSet::default(); + let key = Key::from(1234_u64); + let (_, patch) = Storage::new(&db, &(), 0, true).extend(vec![(key, ValueHash::zero())]); + db.apply_patch(patch); + + let mut patch = WorkingPatchSet::new(1, db.root(0).unwrap()); + let (greatest_leaf, load_result) = patch.load_greatest_key(&db).unwrap(); + assert_eq!(greatest_leaf.full_key, key); + assert_eq!(load_result.longest_prefixes.len(), 1); + assert_eq!(load_result.longest_prefixes[0].nibble_count(), 0); + assert_eq!(load_result.db_reads, 0); + + // Test DB with multiple entries. + let other_key = Key::from_little_endian(&[0xa0; 32]); + let (_, patch) = + Storage::new(&db, &(), 1, true).extend(vec![(other_key, ValueHash::zero())]); + db.apply_patch(patch); + + let mut patch = WorkingPatchSet::new(2, db.root(1).unwrap()); + let (greatest_leaf, load_result) = patch.load_greatest_key(&db).unwrap(); + assert_eq!(greatest_leaf.full_key, other_key); + assert_eq!(load_result.longest_prefixes.len(), 1); + assert_eq!(load_result.longest_prefixes[0].nibble_count(), 1); + assert_eq!(load_result.db_reads, 1); + + let greater_key = Key::from_little_endian(&[0xaf; 32]); + let (_, patch) = + Storage::new(&db, &(), 2, true).extend(vec![(greater_key, ValueHash::zero())]); + db.apply_patch(patch); + + let mut patch = WorkingPatchSet::new(3, db.root(2).unwrap()); + let (greatest_leaf, load_result) = patch.load_greatest_key(&db).unwrap(); + assert_eq!(greatest_leaf.full_key, greater_key); + assert_eq!(load_result.longest_prefixes.len(), 1); + assert_eq!(load_result.longest_prefixes[0].nibble_count(), 2); + assert_eq!(load_result.db_reads, 2); + } } diff --git a/core/lib/merkle_tree/src/storage/proofs.rs b/core/lib/merkle_tree/src/storage/proofs.rs index a9ad624225d1..9e2d172bd6bd 100644 --- a/core/lib/merkle_tree/src/storage/proofs.rs +++ b/core/lib/merkle_tree/src/storage/proofs.rs @@ -215,7 +215,7 @@ impl TreeUpdater { mut root: InternalNode, logs: Vec<(usize, TreeLogEntryWithProof)>, ) -> Vec { - let version = self.patch_set.version(); + let version = self.patch_set.root_version(); let mut root_hash = root.hash(hasher, 0); // Check the kind of each of subtrees. This is used later to ensure the correct @@ -348,6 +348,11 @@ impl<'a, DB: Database + ?Sized> Storage<'a, DB> { root: InternalNode, logs: Vec<(usize, TreeLogEntryWithProof)>, ) -> (BlockOutputWithProofs, PatchSet) { + tracing::debug!( + "Finished updating tree; total leaf count: {}, stats: {:?}", + self.leaf_count, + self.updater.metrics + ); let logs = self.updater.finalize_logs(hasher, root, logs); self.updater.metrics.report(); @@ -499,7 +504,7 @@ mod tests { fn computing_leaf_indices() { let db = prepare_db(); let (instructions, expected_indices) = get_instructions_and_leaf_indices(); - let mut storage = Storage::new(&db, &(), 1); + let mut storage = Storage::new(&db, &(), 1, true); let sorted_keys = SortedKeys::new(instructions.iter().map(|(key, _)| *key)); let parent_nibbles = storage.updater.load_ancestors(&sorted_keys, &db); @@ -511,7 +516,7 @@ mod tests { fn prepare_db() -> PatchSet { let mut db = PatchSet::default(); let (_, patch) = - Storage::new(&db, &(), 0).extend(vec![(byte_key(2), HASH), (byte_key(1), HASH)]); + Storage::new(&db, &(), 0, true).extend(vec![(byte_key(2), HASH), (byte_key(1), HASH)]); db.apply_patch(patch); db } @@ -538,7 +543,7 @@ mod tests { fn extending_storage_with_proofs() { let db = prepare_db(); let (instructions, expected_indices) = get_instructions_and_leaf_indices(); - let storage = Storage::new(&db, &(), 1); + let storage = Storage::new(&db, &(), 1, true); let (block_output, _) = storage.extend_with_proofs(instructions); assert_eq!(block_output.leaf_count, 4); @@ -557,7 +562,7 @@ mod tests { #[test] fn proofs_for_empty_storage() { let db = PatchSet::default(); - let storage = Storage::new(&db, &(), 0); + let storage = Storage::new(&db, &(), 0, true); let instructions = vec![ (byte_key(1), TreeInstruction::Read), (byte_key(2), TreeInstruction::Read), @@ -571,6 +576,6 @@ mod tests { .all(|log| matches!(log.base, TreeLogEntry::ReadMissingKey)); assert!(all_misses); - assert_matches!(patch.roots[&0], Root::Empty); + assert_matches!(patch.patches_by_version[&0].root, Some(Root::Empty)); } } diff --git a/core/lib/merkle_tree/src/storage/rocksdb.rs b/core/lib/merkle_tree/src/storage/rocksdb.rs index b9aca28fd285..da2a7475eae0 100644 --- a/core/lib/merkle_tree/src/storage/rocksdb.rs +++ b/core/lib/merkle_tree/src/storage/rocksdb.rs @@ -194,26 +194,29 @@ impl Database for RocksDBWrapper { patch.manifest.serialize(&mut node_bytes); write_batch.put_cf(tree_cf, Self::MANIFEST_KEY, &node_bytes); - for (root_version, root) in patch.roots { - node_bytes.clear(); - let root_key = NodeKey::empty(root_version); - // Delete the key range corresponding to the entire new version. This removes - // potential garbage left after reverting the tree to a previous version. - let next_root_key = NodeKey::empty(root_version + 1); - let keys_to_delete = &*root_key.to_db_key()..&*next_root_key.to_db_key(); - write_batch.delete_range_cf(tree_cf, keys_to_delete); - - root.serialize(&mut node_bytes); - metrics.update_node_bytes(&Nibbles::EMPTY, &node_bytes); - write_batch.put_cf(tree_cf, &root_key.to_db_key(), &node_bytes); - } - - let all_nodes = patch.nodes_by_version.into_values().flatten(); - for (node_key, node) in all_nodes { - node_bytes.clear(); - node.serialize(&mut node_bytes); - metrics.update_node_bytes(&node_key.nibbles, &node_bytes); - write_batch.put_cf(tree_cf, &node_key.to_db_key(), &node_bytes); + for (version, sub_patch) in patch.patches_by_version { + let is_update = patch.updated_version == Some(version); + let root_key = NodeKey::empty(version); + if !is_update { + // Delete the key range corresponding to the entire new version. This removes + // potential garbage left after reverting the tree to a previous version. + let next_root_key = NodeKey::empty(version + 1); + let keys_to_delete = &*root_key.to_db_key()..&*next_root_key.to_db_key(); + write_batch.delete_range_cf(tree_cf, keys_to_delete); + } + + if let Some(root) = sub_patch.root { + node_bytes.clear(); + root.serialize(&mut node_bytes); + metrics.update_node_bytes(&Nibbles::EMPTY, &node_bytes); + write_batch.put_cf(tree_cf, &root_key.to_db_key(), &node_bytes); + } + for (node_key, node) in sub_patch.nodes { + node_bytes.clear(); + node.serialize(&mut node_bytes); + metrics.update_node_bytes(&node_key.nibbles, &node_bytes); + write_batch.put_cf(tree_cf, &node_key.to_db_key(), &node_bytes); + } } let stale_keys_cf = MerkleTreeColumnFamily::StaleKeys; diff --git a/core/lib/merkle_tree/src/storage/serialization.rs b/core/lib/merkle_tree/src/storage/serialization.rs index 3751f619124f..15d67604cc04 100644 --- a/core/lib/merkle_tree/src/storage/serialization.rs +++ b/core/lib/merkle_tree/src/storage/serialization.rs @@ -200,6 +200,7 @@ impl TreeTags { let mut architecture = None; let mut hasher = None; let mut depth = None; + let mut is_recovering = false; for _ in 0..tag_count { let key = Self::deserialize_str(bytes)?; @@ -216,6 +217,15 @@ impl TreeTags { })?; depth = Some(parsed); } + "is_recovering" => { + let parsed = value.parse::().map_err(|err| { + DeserializeErrorKind::MalformedTag { + name: "is_recovering", + err: err.into(), + } + })?; + is_recovering = parsed; + } _ => return Err(DeserializeErrorKind::UnknownTag(key.to_owned()).into()), } } @@ -223,6 +233,7 @@ impl TreeTags { architecture: architecture.ok_or(DeserializeErrorKind::MissingTag("architecture"))?, hasher: hasher.ok_or(DeserializeErrorKind::MissingTag("hasher"))?, depth: depth.ok_or(DeserializeErrorKind::MissingTag("depth"))?, + is_recovering, }) } @@ -244,13 +255,18 @@ impl TreeTags { } fn serialize(&self, buffer: &mut Vec) { - leb128::write::unsigned(buffer, 3).unwrap(); + let entry_count = 3 + u64::from(self.is_recovering); + leb128::write::unsigned(buffer, entry_count).unwrap(); Self::serialize_str(buffer, "architecture"); Self::serialize_str(buffer, &self.architecture); Self::serialize_str(buffer, "depth"); Self::serialize_str(buffer, &self.depth.to_string()); Self::serialize_str(buffer, "hasher"); Self::serialize_str(buffer, &self.hasher); + if self.is_recovering { + Self::serialize_str(buffer, "is_recovering"); + Self::serialize_str(buffer, "true"); + } } } @@ -300,6 +316,24 @@ mod tests { assert_eq!(manifest_copy, manifest); } + #[test] + fn serializing_manifest_with_recovery_flag() { + let mut manifest = Manifest::new(42, &()); + manifest.tags.as_mut().unwrap().is_recovering = true; + let mut buffer = vec![]; + manifest.serialize(&mut buffer); + assert_eq!(buffer[0], 42); // version count + assert_eq!(buffer[1], 4); // number of tags + assert_eq!( + buffer[2..], + *b"\x0Carchitecture\x06AR16MT\x05depth\x03256\x06hasher\x08no_op256\x0Dis_recovering\x04true" + ); + // ^ length-prefixed tag names and values + + let manifest_copy = Manifest::deserialize(&buffer).unwrap(); + assert_eq!(manifest_copy, manifest); + } + #[test] fn manifest_serialization_errors() { let manifest = Manifest::new(42, &()); diff --git a/core/lib/merkle_tree/src/storage/tests.rs b/core/lib/merkle_tree/src/storage/tests.rs index 64241c05b93f..02ec9d4c800d 100644 --- a/core/lib/merkle_tree/src/storage/tests.rs +++ b/core/lib/merkle_tree/src/storage/tests.rs @@ -12,6 +12,7 @@ use crate::{ hasher::{HasherWithStats, MerklePath}, types::{NodeKey, TreeInstruction, KEY_SIZE}, }; +use zksync_crypto::hasher::blake2::Blake2Hasher; use zksync_types::{H256, U256}; pub(super) const FIRST_KEY: Key = U256([0, 0, 0, 0x_dead_beef_0000_0000]); @@ -35,14 +36,21 @@ pub(super) fn create_patch( nodes: HashMap, ) -> PatchSet { let manifest = Manifest::new(latest_version + 1, &()); - PatchSet::new(manifest, latest_version, root, nodes, vec![]) + PatchSet::new( + manifest, + latest_version, + root, + nodes, + vec![], + Operation::Insert, + ) } #[test] fn inserting_entries_in_empty_database() { let db = PatchSet::default(); let mut updater = TreeUpdater::new(0, Root::Empty); - assert_eq!(updater.patch_set.version(), 0); + assert_eq!(updater.patch_set.root_version(), 0); assert!(updater.patch_set.get(&Nibbles::EMPTY).is_none()); let sorted_keys = SortedKeys::new([FIRST_KEY, SECOND_KEY, THIRD_KEY].into_iter()); @@ -166,7 +174,7 @@ fn changing_child_ref_type() { #[test] fn inserting_node_in_non_empty_database() { let mut db = PatchSet::default(); - let storage = Storage::new(&db, &(), 0); + let storage = Storage::new(&db, &(), 0, true); let kvs = vec![(FIRST_KEY, H256([1; 32])), (SECOND_KEY, H256([2; 32]))]; let (_, patch) = storage.extend(kvs); db.apply_patch(patch); @@ -220,7 +228,7 @@ fn inserting_node_in_non_empty_database() { #[test] fn inserting_node_in_non_empty_database_with_moved_key() { let mut db = PatchSet::default(); - let storage = Storage::new(&db, &(), 0); + let storage = Storage::new(&db, &(), 0, true); let kvs = vec![(FIRST_KEY, H256([1; 32])), (THIRD_KEY, H256([3; 32]))]; let (_, patch) = storage.extend(kvs); db.apply_patch(patch); @@ -290,22 +298,22 @@ fn finalize_merkle_path(mut path: MerklePath, hasher: &HasherWithStats<'_>) -> V #[test] fn reading_keys_does_not_change_child_version() { let mut db = PatchSet::default(); - let storage = Storage::new(&db, &(), 0); + let storage = Storage::new(&db, &(), 0, true); let kvs = vec![(FIRST_KEY, H256([0; 32])), (SECOND_KEY, H256([1; 32]))]; let (_, patch) = storage.extend(kvs); db.apply_patch(patch); - let storage = Storage::new(&db, &(), 1); + let storage = Storage::new(&db, &(), 1, true); let instructions = vec![ (FIRST_KEY, TreeInstruction::Read), (E_KEY, TreeInstruction::Write(H256([2; 32]))), ]; let (_, patch) = storage.extend_with_proofs(instructions); - let Root::Filled { + let Some(Root::Filled { leaf_count, node: Node::Internal(node), - } = &patch.roots[&1] + }) = &patch.patches_by_version[&1].root else { panic!("unexpected root"); }; @@ -317,15 +325,15 @@ fn reading_keys_does_not_change_child_version() { #[test] fn read_ops_are_not_reflected_in_patch() { let mut db = PatchSet::default(); - let storage = Storage::new(&db, &(), 0); + let storage = Storage::new(&db, &(), 0, true); let kvs = vec![(FIRST_KEY, H256([0; 32])), (SECOND_KEY, H256([1; 32]))]; let (_, patch) = storage.extend(kvs); db.apply_patch(patch); - let storage = Storage::new(&db, &(), 1); + let storage = Storage::new(&db, &(), 1, true); let instructions = vec![(FIRST_KEY, TreeInstruction::Read)]; let (_, patch) = storage.extend_with_proofs(instructions); - assert!(patch.nodes_by_version[&1].is_empty()); + assert!(patch.patches_by_version[&1].nodes.is_empty()); } // This maps small indices to keys that differ in the starting nibbles. @@ -339,7 +347,7 @@ fn test_read_instructions_do_not_lead_to_copied_nodes(writes_per_block: u64) { // Write some keys into the database. let mut key_count = writes_per_block; let mut database = PatchSet::default(); - let storage = Storage::new(&database, &(), 0); + let storage = Storage::new(&database, &(), 0, true); let kvs = (0..key_count) .map(|i| (big_endian_key(i), H256::zero())) .collect(); @@ -360,7 +368,7 @@ fn test_read_instructions_do_not_lead_to_copied_nodes(writes_per_block: u64) { instructions.shuffle(&mut rng); key_count += writes_per_block; - let storage = Storage::new(&database, &(), 1); + let storage = Storage::new(&database, &(), 1, true); let (_, patch) = storage.extend_with_proofs(instructions); assert_no_copied_nodes(&database, &patch); database.apply_patch(patch); @@ -368,13 +376,13 @@ fn test_read_instructions_do_not_lead_to_copied_nodes(writes_per_block: u64) { } fn assert_no_copied_nodes(database: &PatchSet, patch: &PatchSet) { - assert_eq!(patch.nodes_by_version.len(), 1); + assert_eq!(patch.patches_by_version.len(), 1); - let (&version, nodes) = patch.nodes_by_version.iter().next().unwrap(); - for (key, node) in nodes { + let (&version, patch) = patch.patches_by_version.iter().next().unwrap(); + for (key, node) in &patch.nodes { let prev_node = (0..version).rev().find_map(|v| { let prev_key = key.nibbles.with_version(v); - database.nodes_by_version[&v].get(&prev_key) + database.patches_by_version[&v].nodes.get(&prev_key) }); if let Some(prev_node) = prev_node { assert_ne!(node, prev_node, "node at {key:?} is copied"); @@ -395,7 +403,7 @@ fn test_replaced_keys_are_correctly_tracked(writes_per_block: usize, with_proofs // Write some keys into the database. let mut database = PatchSet::default(); - let storage = Storage::new(&database, &(), 0); + let storage = Storage::new(&database, &(), 0, true); let kvs = (0..100) .map(|i| (big_endian_key(i), H256::zero())) .collect(); @@ -411,7 +419,7 @@ fn test_replaced_keys_are_correctly_tracked(writes_per_block: usize, with_proofs .into_iter() .map(|i| (big_endian_key(i), H256::zero())); - let storage = Storage::new(&database, &(), new_version); + let storage = Storage::new(&database, &(), new_version, true); let patch = if with_proofs { let instructions = updates.map(|(key, value)| (key, TreeInstruction::Write(value))); storage.extend_with_proofs(instructions.collect()).1 @@ -440,17 +448,18 @@ fn replaced_keys_are_correctly_tracked_with_proofs() { } fn assert_replaced_keys(db: &PatchSet, patch: &PatchSet) { - assert_eq!(patch.nodes_by_version.len(), 1); - let (&version, patch_nodes) = patch.nodes_by_version.iter().next().unwrap(); + assert_eq!(patch.patches_by_version.len(), 1); + let (&version, sub_patch) = patch.patches_by_version.iter().next().unwrap(); assert_eq!(patch.stale_keys_by_version.len(), 1); let replaced_keys = patch.stale_keys_by_version.values().next().unwrap(); - let expected_replaced_keys = patch_nodes.keys().filter_map(|key| { + let expected_replaced_keys = sub_patch.nodes.keys().filter_map(|key| { (0..key.version).rev().find_map(|v| { let prev_key = key.nibbles.with_version(v); let contains_key = db - .nodes_by_version + .patches_by_version .get(&prev_key.version)? + .nodes .contains_key(&prev_key); contains_key.then_some(prev_key) }) @@ -469,18 +478,345 @@ fn tree_handles_keys_at_terminal_level() { let kvs = (0_u32..100) .map(|i| (Key::from(i), ValueHash::zero())) .collect(); - let (_, patch) = Storage::new(&db, &(), 0).extend(kvs); + let (_, patch) = Storage::new(&db, &(), 0, true).extend(kvs); db.apply_patch(patch); // Overwrite a key and check that we don't panic. let new_kvs = vec![(Key::from(0), ValueHash::from_low_u64_be(1))]; - let (_, patch) = Storage::new(&db, &(), 1).extend(new_kvs); + let (_, patch) = Storage::new(&db, &(), 1, true).extend(new_kvs); - assert_eq!(patch.roots[&1].leaf_count(), 100); - assert_eq!(patch.nodes_by_version[&1].len(), 2 * KEY_SIZE); // root is counted separately - for (key, node) in &patch.nodes_by_version[&1] { + assert_eq!( + patch.patches_by_version[&1] + .root + .as_ref() + .unwrap() + .leaf_count(), + 100 + ); + assert_eq!(patch.patches_by_version[&1].nodes.len(), 2 * KEY_SIZE); // root is counted separately + for (key, node) in &patch.patches_by_version[&1].nodes { let is_terminal = key.nibbles.nibble_count() == 2 * KEY_SIZE; assert_eq!(is_terminal, matches!(node, Node::Leaf(_))); } assert_eq!(patch.stale_keys_by_version[&1].len(), 2 * KEY_SIZE + 1); } + +#[test] +fn recovery_flattens_node_versions() { + let recovery_version = 100; + let recovery_entries = (0_u64..10).map(|i| RecoveryEntry { + key: Key::from(i) << 252, // the first key nibbles are distinct + value: ValueHash::zero(), + leaf_index: i + 1, + }); + let patch = Storage::new(&PatchSet::default(), &(), recovery_version, false) + .extend_during_recovery(recovery_entries.collect()); + assert_eq!(patch.patches_by_version.len(), 1); + let (updated_version, patch) = patch.patches_by_version.into_iter().next().unwrap(); + assert_eq!(updated_version, recovery_version); + + let root = patch.root.unwrap(); + assert_eq!(root.leaf_count(), 10); + let Root::Filled { + node: Node::Internal(root_node), + .. + } = &root + else { + panic!("Unexpected root: {root:?}"); + }; + for nibble in 0..10 { + assert_eq!( + root_node.child_ref(nibble).unwrap().version, + recovery_version + ); + let expected_key = Nibbles::single(nibble).with_version(recovery_version); + assert_matches!(patch.nodes[&expected_key], Node::Leaf { .. }); + } +} + +fn test_recovery_with_node_hierarchy(chunk_size: usize) { + let recovery_version = 100; + let recovery_entries = (0_u64..256).map(|i| RecoveryEntry { + key: Key::from(i) << 248, // the first two key nibbles are distinct + value: ValueHash::zero(), + leaf_index: i + 1, + }); + let recovery_entries: Vec<_> = recovery_entries.collect(); + + let mut db = PatchSet::default(); + for recovery_chunk in recovery_entries.chunks(chunk_size) { + let patch = Storage::new(&db, &(), recovery_version, false) + .extend_during_recovery(recovery_chunk.to_vec()); + db.apply_patch(patch); + } + assert_eq!(db.updated_version, Some(recovery_version)); + let patch = db.patches_by_version.remove(&recovery_version).unwrap(); + + let root = patch.root.unwrap(); + assert_eq!(root.leaf_count(), 256); + let Root::Filled { + node: Node::Internal(root_node), + .. + } = &root + else { + panic!("Unexpected root: {root:?}"); + }; + + for nibble in 0..16 { + let child_ref = root_node.child_ref(nibble).unwrap(); + assert!(!child_ref.is_leaf); + assert_eq!(child_ref.version, recovery_version); + + let internal_node_key = Nibbles::single(nibble).with_version(recovery_version); + let node = &patch.nodes[&internal_node_key]; + let Node::Internal(node) = node else { + panic!("Unexpected upper-level node: {node:?}"); + }; + assert_eq!(node.child_count(), 16); + + for (second_nibble, child_ref) in node.children() { + let i = nibble * 16 + second_nibble; + assert!(child_ref.is_leaf); + assert_eq!(child_ref.version, recovery_version); + let leaf_key = Nibbles::new(&(Key::from(i) << 248), 2).with_version(recovery_version); + assert_matches!(patch.nodes[&leaf_key], Node::Leaf { .. }); + } + } +} + +#[test] +fn recovery_with_node_hierarchy() { + test_recovery_with_node_hierarchy(256); // single chunk + for chunk_size in [4, 5, 20, 69, 127, 128] { + println!("Testing recovery with chunk size {chunk_size}"); + test_recovery_with_node_hierarchy(chunk_size); + } +} + +fn test_recovery_with_deep_node_hierarchy(chunk_size: usize) { + let recovery_version = 1_000; + let recovery_entries = (0_u64..256).map(|i| RecoveryEntry { + key: Key::from(i), // the last two key nibbles are distinct + value: ValueHash::zero(), + leaf_index: i + 1, + }); + let recovery_entries: Vec<_> = recovery_entries.collect(); + + let mut db = PatchSet::default(); + for recovery_chunk in recovery_entries.chunks(chunk_size) { + let patch = Storage::new(&db, &(), recovery_version, false) + .extend_during_recovery(recovery_chunk.to_vec()); + db.apply_patch(patch); + } + let mut patch = db.patches_by_version.remove(&recovery_version).unwrap(); + // Manually remove all stale keys from the patch + assert_eq!(db.stale_keys_by_version.len(), 1); + for stale_key in &db.stale_keys_by_version[&recovery_version] { + assert!( + patch.nodes.remove(stale_key).is_some(), + "Stale key {stale_key} is missing" + ); + } + + let root = patch.root.unwrap(); + assert_eq!(root.leaf_count(), 256); + let Root::Filled { + node: Node::Internal(root_node), + .. + } = &root + else { + panic!("Unexpected root: {root:?}"); + }; + assert_eq!(root_node.child_count(), 1); + let child_ref = root_node.child_ref(0).unwrap(); + assert!(!child_ref.is_leaf); + assert_eq!(child_ref.version, recovery_version); + + for (node_key, node) in patch.nodes { + assert_eq!( + node_key.version, recovery_version, + "Unexpected version for {node_key}" + ); + + let nibble_count = node_key.nibbles.nibble_count(); + if nibble_count < 64 { + let Node::Internal(node) = node else { + panic!("Unexpected node at {node_key}: {node:?}"); + }; + assert_eq!(node.child_count(), if nibble_count < 62 { 1 } else { 16 }); + } else { + assert_matches!( + node, + Node::Leaf(_), + "Unexpected node at {node_key}: {node:?}" + ); + } + } +} + +#[test] +fn recovery_with_deep_node_hierarchy() { + test_recovery_with_deep_node_hierarchy(256); + for chunk_size in [5, 7, 20, 59, 127, 128] { + println!("Testing recovery with chunk size {chunk_size}"); + test_recovery_with_deep_node_hierarchy(chunk_size); + } +} + +#[test] +fn recovery_workflow_with_multiple_stages() { + let mut db = PatchSet::default(); + let recovery_version = 100; + let recovery_entries = (0_u64..100).map(|i| RecoveryEntry { + key: Key::from(i), + value: ValueHash::zero(), + leaf_index: i, + }); + let patch = Storage::new(&db, &(), recovery_version, false) + .extend_during_recovery(recovery_entries.collect()); + assert_eq!(patch.root(recovery_version).unwrap().leaf_count(), 100); + db.apply_patch(patch); + + let more_recovery_entries = (100_u64..200).map(|i| RecoveryEntry { + key: Key::from(i), + value: ValueHash::zero(), + leaf_index: i, + }); + + let patch = Storage::new(&db, &(), recovery_version, false) + .extend_during_recovery(more_recovery_entries.collect()); + assert_eq!(patch.root(recovery_version).unwrap().leaf_count(), 200); + db.apply_patch(patch); + + // Check that all entries can be accessed + let storage = Storage::new(&db, &(), recovery_version + 1, true); + let instructions = (0_u32..200).map(|i| (Key::from(i), TreeInstruction::Read)); + let (output, _) = storage.extend_with_proofs(instructions.collect()); + assert_eq!(output.leaf_count, 200); + assert_eq!(output.logs.len(), 200); + assert!(output + .logs + .iter() + .all(|log| matches!(log.base, TreeLogEntry::Read { .. }))); +} + +fn test_recovery_pruning_equivalence( + chunk_size: usize, + recovery_chunk_size: usize, + hasher: &dyn HashTree, +) { + const RNG_SEED: u64 = 123; + + println!( + "Testing recovery–pruning equivalence (chunk size: {chunk_size}, recovery chunk size: \ + {recovery_chunk_size})" + ); + + let mut rng = StdRng::seed_from_u64(RNG_SEED); + let kvs = (0..100).map(|i| { + ( + U256([rng.gen(), rng.gen(), rng.gen(), rng.gen()]), + ValueHash::repeat_byte(i), + ) + }); + let kvs: Vec<_> = kvs.collect(); + + // Add `kvs` into the tree in several commits. + let mut db = PatchSet::default(); + for (version, chunk) in kvs.chunks(chunk_size).enumerate() { + let (_, patch) = Storage::new(&db, hasher, version as u64, true).extend(chunk.to_vec()); + db.apply_patch(patch); + } + // Unite all remaining nodes to a map and manually remove all stale keys. + let recovered_version = db.manifest.version_count - 1; + let mut root = db.root(recovered_version).unwrap(); + let mut all_nodes: HashMap<_, _> = db + .patches_by_version + .into_values() + .flat_map(|sub_patch| sub_patch.nodes) + .collect(); + for stale_key in db.stale_keys_by_version.values().flatten() { + all_nodes.remove(stale_key); + } + + // Generate recovery entries. + let recovery_entries = all_nodes.values().filter_map(|node| { + if let Node::Leaf(leaf) = node { + return Some(RecoveryEntry { + key: leaf.full_key, + value: leaf.value_hash, + leaf_index: leaf.leaf_index, + }); + } + None + }); + let mut recovery_entries: Vec<_> = recovery_entries.collect(); + assert_eq!(recovery_entries.len(), 100); + recovery_entries.sort_unstable_by_key(|entry| entry.key); + + // Recover the tree. + let mut recovered_db = PatchSet::default(); + for recovery_chunk in recovery_entries.chunks(recovery_chunk_size) { + let patch = Storage::new(&recovered_db, hasher, recovered_version, false) + .extend_during_recovery(recovery_chunk.to_vec()); + recovered_db.apply_patch(patch); + } + let sub_patch = recovered_db + .patches_by_version + .remove(&recovered_version) + .unwrap(); + let recovered_root = sub_patch.root.unwrap(); + let mut all_recovered_nodes = sub_patch.nodes; + for stale_key in db.stale_keys_by_version.values().flatten() { + all_recovered_nodes.remove(stale_key); + } + + // Nodes must be identical for the pruned and recovered trees up to the version. + if let Root::Filled { + node: Node::Internal(node), + .. + } = &mut root + { + for child_ref in node.child_refs_mut() { + child_ref.version = recovered_version; + } + } + assert_eq!(recovered_root, root); + + let flattened_version_nodes: HashMap<_, _> = all_nodes + .into_iter() + .map(|(key, mut node)| { + if let Node::Internal(node) = &mut node { + for child_ref in node.child_refs_mut() { + child_ref.version = recovered_version; + } + } + (key.nibbles.with_version(recovered_version), node) + }) + .collect(); + assert_eq!(all_recovered_nodes, flattened_version_nodes); +} + +#[test] +fn recovery_pruning_equivalence() { + for chunk_size in [3, 5, 7, 11, 21, 42, 99, 100] { + // No chunking during recovery (simple case). + test_recovery_pruning_equivalence(chunk_size, 100, &()); + // Recovery is chunked (more complex case). + for recovery_chunk_size in [chunk_size, 1, 6, 19, 50, 73] { + test_recovery_pruning_equivalence(chunk_size, recovery_chunk_size, &()); + } + } +} + +#[test] +fn recovery_pruning_equivalence_with_hashing() { + for chunk_size in [3, 7, 21, 42, 100] { + // No chunking during recovery (simple case). + test_recovery_pruning_equivalence(chunk_size, 100, &Blake2Hasher); + // Recovery is chunked (more complex case). + for recovery_chunk_size in [chunk_size, 1, 19, 73] { + test_recovery_pruning_equivalence(chunk_size, recovery_chunk_size, &Blake2Hasher); + } + } +} diff --git a/core/lib/merkle_tree/src/types/internal.rs b/core/lib/merkle_tree/src/types/internal.rs index 86568da7f5de..5e875f6e28ac 100644 --- a/core/lib/merkle_tree/src/types/internal.rs +++ b/core/lib/merkle_tree/src/types/internal.rs @@ -25,6 +25,7 @@ pub(crate) struct TreeTags { pub architecture: String, pub depth: usize, pub hasher: String, + pub is_recovering: bool, } impl TreeTags { @@ -35,10 +36,11 @@ impl TreeTags { architecture: Self::ARCHITECTURE.to_owned(), hasher: hasher.name().to_owned(), depth: TREE_DEPTH, + is_recovering: false, } } - pub fn assert_consistency(&self, hasher: &dyn HashTree) { + pub fn assert_consistency(&self, hasher: &dyn HashTree, expecting_recovery: bool) { assert_eq!( self.architecture, Self::ARCHITECTURE, @@ -59,6 +61,18 @@ impl TreeTags { hasher.name(), self.hasher ); + + if expecting_recovery { + assert!( + self.is_recovering, + "Tree is expected to be in the process of recovery, but it is not" + ); + } else { + assert!( + !self.is_recovering, + "Tree is being recovered; cannot access it until recovery finishes" + ); + } } } @@ -205,6 +219,26 @@ impl Nibbles { } Some(child) } + + /// Returns nibbles that form a common prefix between these nibbles and the provided `key`. + pub fn common_prefix(mut self, other: &Self) -> Self { + for i in 0..(self.nibble_count + 1) / 2 { + let (this_byte, other_byte) = (self.bytes[i], other.bytes[i]); + if this_byte != other_byte { + // Check whether the first nibble matches. + if this_byte & 0xf0 == other_byte & 0xf0 { + self.nibble_count = i * 2 + 1; + self.bytes[i] &= 0xf0; + self.bytes[(i + 1)..].fill(0); + } else { + self.nibble_count = i * 2; + self.bytes[i..].fill(0); + } + return self; + } + } + self + } } impl fmt::Display for Nibbles { @@ -409,6 +443,16 @@ impl InternalNode { self.children.values() } + #[cfg(test)] + pub(crate) fn child_refs_mut(&mut self) -> impl Iterator + '_ { + self.children.values_mut() + } + + pub(crate) fn last_child_ref(&self) -> (u8, &ChildRef) { + self.children.last().unwrap() + // ^ `unwrap()` is safe by construction; all persisted internal nodes are not empty + } + pub(crate) fn child_hashes(&self) -> [Option; Self::CHILD_COUNT as usize] { let mut hashes = [None; Self::CHILD_COUNT as usize]; for (nibble, child_ref) in self.children.iter() { @@ -561,6 +605,34 @@ mod tests { assert!(nibbles.push(0xb).is_none()); } + #[test] + fn nibbles_prefix() { + let nibbles = Nibbles::new(&TEST_KEY, 6); + assert_eq!(nibbles.common_prefix(&nibbles), nibbles); + + let nibbles = Nibbles::new(&TEST_KEY, 6); + let prefix = Nibbles::new(&TEST_KEY, 4); + assert_eq!(nibbles.common_prefix(&prefix), prefix); + assert_eq!(prefix.common_prefix(&nibbles), prefix); + + let nibbles = Nibbles::new(&TEST_KEY, 7); + assert_eq!(nibbles.common_prefix(&prefix), prefix); + assert_eq!(prefix.common_prefix(&nibbles), prefix); + + let nibbles = Nibbles::new(&TEST_KEY, 64); + let diverging_nibbles = Nibbles::new(&TEST_KEY, 4).push(0x1).unwrap(); + assert_eq!(nibbles.common_prefix(&diverging_nibbles), prefix); + + let diverging_nibbles = Nibbles::new(&TEST_KEY, 5).push(0x1).unwrap(); + assert_eq!( + nibbles.common_prefix(&diverging_nibbles), + Nibbles::new(&TEST_KEY, 5) + ); + + let diverging_nibbles = Nibbles::from_parts([0xff; KEY_SIZE], 64); + assert_eq!(nibbles.common_prefix(&diverging_nibbles), Nibbles::EMPTY); + } + #[test] fn node_key_serialization() { let nibbles = Nibbles::new(&TEST_KEY, 6); diff --git a/core/lib/merkle_tree/src/utils.rs b/core/lib/merkle_tree/src/utils.rs index 5faedf597162..9542b24bbd3c 100644 --- a/core/lib/merkle_tree/src/utils.rs +++ b/core/lib/merkle_tree/src/utils.rs @@ -63,6 +63,13 @@ impl SmallMap { Self::indices(self.bitmap).zip(&self.values) } + pub fn last(&self) -> Option<(u8, &V)> { + let greatest_set_bit = (u16::BITS - self.bitmap.leading_zeros()).checked_sub(1)?; + let greatest_set_bit = u8::try_from(greatest_set_bit).unwrap(); + // ^ `unwrap()` is safe by construction: `greatest_set_bit <= 15`. + Some((greatest_set_bit, self.values.last()?)) + } + fn indices(bitmap: u16) -> impl Iterator { (0..Self::CAPACITY).filter(move |&index| { let mask = 1 << u16::from(index); @@ -74,6 +81,11 @@ impl SmallMap { self.values.iter() } + #[cfg(test)] + pub fn values_mut(&mut self) -> impl Iterator + '_ { + self.values.iter_mut() + } + pub fn get_mut(&mut self, index: u8) -> Option<&mut V> { assert!(index < Self::CAPACITY, "index is too large"); diff --git a/core/lib/merkle_tree/tests/integration/common.rs b/core/lib/merkle_tree/tests/integration/common.rs index dff7c8ca012c..fd9e00855c20 100644 --- a/core/lib/merkle_tree/tests/integration/common.rs +++ b/core/lib/merkle_tree/tests/integration/common.rs @@ -1,5 +1,11 @@ //! Shared functionality. +use once_cell::sync::Lazy; + +use std::collections::HashMap; + +use zksync_crypto::hasher::{blake2::Blake2Hasher, Hasher}; +use zksync_merkle_tree::{HashTree, TreeInstruction}; use zksync_types::{AccountTreeId, Address, StorageKey, H256, U256}; pub fn generate_key_value_pairs(indexes: impl Iterator) -> Vec<(U256, H256)> { @@ -11,3 +17,102 @@ pub fn generate_key_value_pairs(indexes: impl Iterator) -> Vec<(U256 }); kvs.collect() } + +pub fn compute_tree_hash(kvs: impl Iterator) -> H256 { + let kvs_with_indices = kvs + .enumerate() + .map(|(i, (key, value))| (key, value, i as u64 + 1)); + compute_tree_hash_with_indices(kvs_with_indices) +} + +// The extended version of computations used in `InternalNode`. +fn compute_tree_hash_with_indices(kvs: impl Iterator) -> H256 { + let hasher = Blake2Hasher; + let mut empty_tree_hash = hasher.hash_bytes(&[0_u8; 40]); + let level = kvs.map(|(key, value, leaf_index)| { + let mut bytes = [0_u8; 40]; + bytes[..8].copy_from_slice(&leaf_index.to_be_bytes()); + bytes[8..].copy_from_slice(value.as_ref()); + (key, hasher.hash_bytes(&bytes)) + }); + let mut level: Vec<(U256, H256)> = level.collect(); + if level.is_empty() { + return hasher.empty_subtree_hash(256); + } + level.sort_unstable_by_key(|(key, _)| *key); + + for _ in 0..256 { + let mut next_level = vec![]; + let mut i = 0; + while i < level.len() { + let (pos, hash) = level[i]; + let aggregate_hash = if pos.bit(0) { + // `pos` corresponds to a right branch of its parent + hasher.compress(&empty_tree_hash, &hash) + } else if let Some((next_pos, next_hash)) = level.get(i + 1) { + if pos + 1 == *next_pos { + i += 1; + hasher.compress(&hash, next_hash) + } else { + hasher.compress(&hash, &empty_tree_hash) + } + } else { + hasher.compress(&hash, &empty_tree_hash) + }; + next_level.push((pos >> 1, aggregate_hash)); + i += 1; + } + + level = next_level; + empty_tree_hash = hasher.compress(&empty_tree_hash, &empty_tree_hash); + } + level[0].1 +} + +// Computing the expected hash takes some time in the debug mode, so we memoize it. +pub static KVS_AND_HASH: Lazy<(Vec<(U256, H256)>, H256)> = Lazy::new(|| { + let kvs = generate_key_value_pairs(0..100); + let expected_hash = compute_tree_hash(kvs.iter().copied()); + (kvs, expected_hash) +}); + +pub fn convert_to_writes(kvs: &[(U256, H256)]) -> Vec<(U256, TreeInstruction)> { + let kvs = kvs + .iter() + .map(|&(key, hash)| (key, TreeInstruction::Write(hash))); + kvs.collect() +} + +/// Emulates leaf index assignment in a real Merkle tree. +#[derive(Debug)] +pub struct TreeMap(HashMap); + +impl TreeMap { + pub fn new(initial_entries: &[(U256, H256)]) -> Self { + let map = initial_entries + .iter() + .enumerate() + .map(|(i, (key, value))| (*key, (*value, i as u64 + 1))) + .collect(); + Self(map) + } + + pub fn extend(&mut self, kvs: &[(U256, H256)]) { + for &(key, new_value) in kvs { + if let Some((value, _)) = self.0.get_mut(&key) { + *value = new_value; + } else { + let leaf_index = self.0.len() as u64 + 1; + self.0.insert(key, (new_value, leaf_index)); + } + } + } + + pub fn root_hash(&self) -> H256 { + let entries = self + .0 + .iter() + .map(|(key, (value, idx))| (*key, *value, *idx)); + compute_tree_hash_with_indices(entries) + } +} diff --git a/core/lib/merkle_tree/tests/integration/main.rs b/core/lib/merkle_tree/tests/integration/main.rs index bf3391df6ccf..a6afb1a58c7a 100644 --- a/core/lib/merkle_tree/tests/integration/main.rs +++ b/core/lib/merkle_tree/tests/integration/main.rs @@ -4,3 +4,4 @@ mod common; mod consistency; mod domain; mod merkle_tree; +mod recovery; diff --git a/core/lib/merkle_tree/tests/integration/merkle_tree.rs b/core/lib/merkle_tree/tests/integration/merkle_tree.rs index f94335390eea..ad9467b8e5f8 100644 --- a/core/lib/merkle_tree/tests/integration/merkle_tree.rs +++ b/core/lib/merkle_tree/tests/integration/merkle_tree.rs @@ -1,69 +1,17 @@ //! Tests not tied to the zksync domain. -use once_cell::sync::Lazy; use rand::{rngs::StdRng, seq::SliceRandom, Rng, SeedableRng}; use std::{cmp, mem}; -use zksync_crypto::hasher::{blake2::Blake2Hasher, Hasher}; +use zksync_crypto::hasher::blake2::Blake2Hasher; use zksync_merkle_tree::{ Database, HashTree, MerkleTree, PatchSet, Patched, TreeInstruction, TreeLogEntry, TreeRangeDigest, }; use zksync_types::{AccountTreeId, Address, StorageKey, H256, U256}; -use crate::common::generate_key_value_pairs; - -fn convert_to_writes(kvs: &[(U256, H256)]) -> Vec<(U256, TreeInstruction)> { - let kvs = kvs - .iter() - .map(|&(key, hash)| (key, TreeInstruction::Write(hash))); - kvs.collect() -} - -// The extended version of computations used in `InternalNode`. -fn compute_tree_hash(kvs: &[(U256, H256)]) -> H256 { - assert!(!kvs.is_empty()); - - let hasher = Blake2Hasher; - let mut empty_tree_hash = hasher.hash_bytes(&[0_u8; 40]); - let level = kvs.iter().enumerate().map(|(i, (key, value))| { - let leaf_index = i as u64 + 1; - let mut bytes = [0_u8; 40]; - bytes[..8].copy_from_slice(&leaf_index.to_be_bytes()); - bytes[8..].copy_from_slice(value.as_ref()); - (*key, hasher.hash_bytes(&bytes)) - }); - let mut level: Vec<(U256, H256)> = level.collect(); - level.sort_unstable_by_key(|(key, _)| *key); - - for _ in 0..256 { - let mut next_level = vec![]; - let mut i = 0; - while i < level.len() { - let (pos, hash) = level[i]; - let aggregate_hash = if pos.bit(0) { - // `pos` corresponds to a right branch of its parent - hasher.compress(&empty_tree_hash, &hash) - } else if let Some((next_pos, next_hash)) = level.get(i + 1) { - if pos + 1 == *next_pos { - i += 1; - hasher.compress(&hash, next_hash) - } else { - hasher.compress(&hash, &empty_tree_hash) - } - } else { - hasher.compress(&hash, &empty_tree_hash) - }; - next_level.push((pos >> 1, aggregate_hash)); - i += 1; - } - - level = next_level; - empty_tree_hash = hasher.compress(&empty_tree_hash, &empty_tree_hash); - } - level[0].1 -} +use crate::common::{compute_tree_hash, convert_to_writes, generate_key_value_pairs, KVS_AND_HASH}; #[test] fn compute_tree_hash_works_correctly() { @@ -76,7 +24,7 @@ fn compute_tree_hash_works_correctly() { let address: Address = "4b3af74f66ab1f0da3f2e4ec7a3cb99baf1af7b2".parse().unwrap(); let key = StorageKey::new(AccountTreeId::new(address), H256::zero()); let key = key.hashed_key_u256(); - let hash = compute_tree_hash(&[(key, H256([1; 32]))]); + let hash = compute_tree_hash([(key, H256([1; 32]))].into_iter()); assert_eq!(hash, EXPECTED_HASH); } @@ -87,7 +35,7 @@ fn root_hash_is_computed_correctly_on_empty_tree() { let mut tree = MerkleTree::new(PatchSet::default()); let kvs = generate_key_value_pairs(0..kv_count); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); let output = tree.extend(kvs); assert_eq!(output.root_hash, expected_hash); } @@ -104,7 +52,7 @@ fn output_proofs_are_computed_correctly_on_empty_tree() { let mut tree = MerkleTree::new(PatchSet::default()); let kvs = generate_key_value_pairs(0..kv_count); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); let instructions = convert_to_writes(&kvs); let output = tree.extend_with_proofs(instructions.clone()); @@ -134,7 +82,7 @@ fn entry_proofs_are_computed_correctly_on_empty_tree() { let mut tree = MerkleTree::new(PatchSet::default()); let kvs = generate_key_value_pairs(0..kv_count); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); tree.extend(kvs.clone()); let existing_keys: Vec<_> = kvs.iter().map(|(key, _)| *key).collect(); @@ -182,7 +130,7 @@ fn proofs_are_computed_correctly_for_mixed_instructions() { let mut instructions: Vec<_> = reads.collect(); // Overwrite all keys in the tree. let writes: Vec<_> = kvs.iter().map(|(key, _)| (*key, H256::zero())).collect(); - let expected_hash = compute_tree_hash(&writes); + let expected_hash = compute_tree_hash(writes.iter().copied()); instructions.extend(convert_to_writes(&writes)); instructions.shuffle(&mut rng); @@ -221,13 +169,6 @@ fn proofs_are_computed_correctly_for_missing_keys() { output.verify_proofs(&Blake2Hasher, empty_tree_hash, &instructions); } -// Computing the expected hash takes some time in the debug mode, so we memoize it. -static KVS_AND_HASH: Lazy<(Vec<(U256, H256)>, H256)> = Lazy::new(|| { - let kvs = generate_key_value_pairs(0..100); - let expected_hash = compute_tree_hash(&kvs); - (kvs, expected_hash) -}); - fn test_intermediate_commits(db: &mut impl Database, chunk_size: usize) { let (kvs, expected_hash) = &*KVS_AND_HASH; let mut final_hash = H256::zero(); @@ -374,7 +315,7 @@ fn test_root_hash_computing_with_key_updates(db: impl Database) { let mut kvs = generate_key_value_pairs(0..50); let mut tree = MerkleTree::new(db); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); let output = tree.extend(kvs.clone()); assert_eq!(output.root_hash, expected_hash); @@ -389,7 +330,7 @@ fn test_root_hash_computing_with_key_updates(db: impl Database) { let changed_kvs: Vec<_> = changed_kvs.collect(); let new_kvs = generate_key_value_pairs(50..75); kvs.extend_from_slice(&new_kvs); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); // We can merge `changed_kvs` and `new_kvs` in any way that preserves `new_kvs` ordering. // We'll do multiple ways (which also will effectively test DB rollbacks). @@ -504,7 +445,7 @@ fn test_root_hash_equals_to_previous_implementation(db: &mut impl Database) { let values = (0..100).map(H256::from_low_u64_be); let kvs: Vec<_> = keys.zip(values).collect(); - let expected_hash = compute_tree_hash(&kvs); + let expected_hash = compute_tree_hash(kvs.iter().copied()); assert_eq!(expected_hash, PREV_IMPL_HASH); let mut tree = MerkleTree::new(db); @@ -618,7 +559,7 @@ mod rocksdb { use std::collections::BTreeMap; use super::*; - use zksync_merkle_tree::{MerkleTreeColumnFamily, RocksDBWrapper}; + use zksync_merkle_tree::{MerkleTreeColumnFamily, MerkleTreePruner, RocksDBWrapper}; use zksync_storage::RocksDB; #[derive(Debug)] @@ -683,14 +624,34 @@ mod rocksdb { let raw_db = db.into_inner(); let snapshot_name = format!("db-snapshot-{chunk_size}-chunked-commits"); insta::assert_yaml_snapshot!(snapshot_name, DatabaseSnapshot::new(&raw_db)); + db = clean_db(raw_db); + } + } - // Clear the entire database instead of using `MerkleTree::truncate_versions()` - // so that it doesn't contain any junk that can influence snapshots. - let mut batch = raw_db.new_write_batch(); - let cf = MerkleTreeColumnFamily::Tree; - batch.delete_range_cf(cf, (&[] as &[_])..&u64::MAX.to_be_bytes()); - raw_db.write(batch).unwrap(); - db = RocksDBWrapper::from(raw_db); + fn clean_db(raw_db: RocksDB) -> RocksDBWrapper { + // Clear the entire database instead of using `MerkleTree::truncate_versions()` + // so that it doesn't contain any junk that can influence snapshots. + let mut batch = raw_db.new_write_batch(); + let cf = MerkleTreeColumnFamily::Tree; + batch.delete_range_cf(cf, (&[] as &[_])..&u64::MAX.to_be_bytes()); + raw_db.write(batch).unwrap(); + RocksDBWrapper::from(raw_db) + } + + #[test] + fn snapshot_for_pruned_tree() { + let Harness { mut db, dir: _dir } = Harness::new(); + for chunk_size in [3, 8, 21] { + test_intermediate_commits(&mut db, chunk_size); + let (mut pruner, _) = MerkleTreePruner::new(&mut db, 0); + pruner.run_once(); + + let raw_db = db.into_inner(); + let snapshot_name = format!("db-snapshot-{chunk_size}-chunked-commits-pruned"); + let db_snapshot = DatabaseSnapshot::new(&raw_db); + assert!(db_snapshot.stale_keys.is_empty()); + insta::assert_yaml_snapshot!(snapshot_name, db_snapshot); + db = clean_db(raw_db); } } diff --git a/core/lib/merkle_tree/tests/integration/recovery.rs b/core/lib/merkle_tree/tests/integration/recovery.rs new file mode 100644 index 000000000000..fe89dded5c32 --- /dev/null +++ b/core/lib/merkle_tree/tests/integration/recovery.rs @@ -0,0 +1,141 @@ +//! Tests for tree recovery. + +use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng}; +use zksync_crypto::hasher::blake2::Blake2Hasher; + +use zksync_merkle_tree::{ + recovery::{MerkleTreeRecovery, RecoveryEntry}, + Database, MerkleTree, PatchSet, PruneDatabase, ValueHash, +}; + +use crate::common::{convert_to_writes, generate_key_value_pairs, TreeMap, KVS_AND_HASH}; + +#[test] +fn recovery_basics() { + let (kvs, expected_hash) = &*KVS_AND_HASH; + let recovery_entries = kvs + .iter() + .enumerate() + .map(|(i, &(key, value))| RecoveryEntry { + key, + value, + leaf_index: i as u64 + 1, + }); + let mut recovery_entries: Vec<_> = recovery_entries.collect(); + recovery_entries.sort_unstable_by_key(|entry| entry.key); + let greatest_key = recovery_entries[99].key; + + let recovered_version = 123; + let mut recovery = MerkleTreeRecovery::new(PatchSet::default(), recovered_version); + recovery.extend(recovery_entries); + + assert_eq!(recovery.last_processed_key(), Some(greatest_key)); + assert_eq!(recovery.root_hash(), *expected_hash); + + let tree = recovery.finalize(); + tree.verify_consistency(recovered_version).unwrap(); +} + +fn test_recovery_in_chunks(mut create_db: impl FnMut() -> DB) { + let (kvs, expected_hash) = &*KVS_AND_HASH; + let recovery_entries = kvs + .iter() + .enumerate() + .map(|(i, &(key, value))| RecoveryEntry { + key, + value, + leaf_index: i as u64 + 1, + }); + let mut recovery_entries: Vec<_> = recovery_entries.collect(); + recovery_entries.sort_unstable_by_key(|entry| entry.key); + let greatest_key = recovery_entries[99].key; + + let recovered_version = 123; + for chunk_size in [6, 10, 17, 42] { + let mut db = create_db(); + let mut recovery = MerkleTreeRecovery::new(&mut db, recovered_version); + for (i, chunk) in recovery_entries.chunks(chunk_size).enumerate() { + recovery.extend(chunk.to_vec()); + if i % 3 == 1 { + recovery = MerkleTreeRecovery::new(&mut db, recovered_version); + // ^ Simulate recovery interruption and restart + } + } + + assert_eq!(recovery.last_processed_key(), Some(greatest_key)); + assert_eq!(recovery.root_hash(), *expected_hash); + + let mut tree = recovery.finalize(); + tree.verify_consistency(recovered_version).unwrap(); + // Check that new tree versions can be built and function as expected. + test_tree_after_recovery(&mut tree, recovered_version, *expected_hash); + } +} + +fn test_tree_after_recovery( + tree: &mut MerkleTree, + recovered_version: u64, + root_hash: ValueHash, +) { + const RNG_SEED: u64 = 765; + const CHUNK_SIZE: usize = 18; + + assert_eq!(tree.latest_version(), Some(recovered_version)); + assert_eq!(tree.root_hash(recovered_version), Some(root_hash)); + for ver in 0..recovered_version { + assert_eq!(tree.root_hash(ver), None); + } + + // Check adding new and updating existing entries in the tree. + let mut rng = StdRng::seed_from_u64(RNG_SEED); + let mut kvs = generate_key_value_pairs(100..=150); + let mut modified_kvs = generate_key_value_pairs(50..=100); + for (_, value) in &mut modified_kvs { + *value = ValueHash::repeat_byte(1); + } + kvs.extend(modified_kvs); + kvs.shuffle(&mut rng); + + let mut tree_map = TreeMap::new(&KVS_AND_HASH.0); + let mut prev_root_hash = root_hash; + for (i, chunk) in kvs.chunks(CHUNK_SIZE).enumerate() { + tree_map.extend(chunk); + + let new_root_hash = if i % 2 == 0 { + let output = tree.extend(chunk.to_vec()); + output.root_hash + } else { + let instructions = convert_to_writes(chunk); + let output = tree.extend_with_proofs(instructions.clone()); + output.verify_proofs(&Blake2Hasher, prev_root_hash, &instructions); + output.root_hash().unwrap() + }; + + assert_eq!(new_root_hash, tree_map.root_hash()); + tree.verify_consistency(recovered_version + i as u64) + .unwrap(); + prev_root_hash = new_root_hash; + } +} + +#[test] +fn recovery_in_chunks() { + test_recovery_in_chunks(PatchSet::default); +} + +mod rocksdb { + use tempfile::TempDir; + + use super::*; + use zksync_merkle_tree::RocksDBWrapper; + + #[test] + fn recovery_in_chunks() { + let temp_dir = TempDir::new().unwrap(); + let mut counter = 0; + test_recovery_in_chunks(|| { + counter += 1; + RocksDBWrapper::new(&temp_dir.path().join(counter.to_string())) + }); + } +} diff --git a/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-21-chunked-commits-pruned.snap b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-21-chunked-commits-pruned.snap new file mode 100644 index 000000000000..b8463049d616 --- /dev/null +++ b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-21-chunked-commits-pruned.snap @@ -0,0 +1,136 @@ +--- +source: core/lib/merkle_tree/tests/integration/merkle_tree.rs +assertion_line: 658 +expression: "DatabaseSnapshot::new(&raw_db)" +--- +tree: + "00": 05030c61726368697465637475726506415231364d5405646570746803323536066861736865720a626c616b653273323536 + "00000000000000000200": 0059914596ed2e70745c44ef315747ea29aff72bce6757aca30a434d9bb70781000000000000000000000000000000000000000000000000000000000000000202 + "00000000000000000204": 049dfddb9f03237fed8d902e350f0b0e9b85af93f49fe02f0d189c47cd7b122b000000000000000000000000000000000000000000000000000000000000000707 + 0000000000000000020e: 0edeb726b4588b4b373d38b7b43e3083c6e105c991e1812bdcce6c373ed98dd5000000000000000000000000000000000000000000000000000000000000000a0a + "00000000000000000229": 297faa5f83989e2a8b574589c3eeb5fdced0634fd75d662507604796dd4edd1b000000000000000000000000000000000000000000000000000000000000001515 + 0000000000000000022b: 2b5ffd54bff018009f93f7cd782e8922b24520eb99a100efe2c1a4ae6c1ca3f5000000000000000000000000000000000000000000000000000000000000000b0b + 0000000000000000027c: 7cabd5f57ff72670052939485c4533b01b26fd0e490c31bd48b5ac5002ff1f83000000000000000000000000000000000000000000000000000000000000000909 + 0000000000000000027e: 7ed4dea78574266e019059e5b5fd6f94ed1632bd4a643d1c51aa02974def5684000000000000000000000000000000000000000000000000000000000000000303 + 000000000000000002a5: a58e3a77937b9b747f70d4aee9e84992d7955e52e2de71dc9615df3d16b2b816000000000000000000000000000000000000000000000000000000000000001313 + 000000000000000002ab: ab314b8d202e718d011f9f90df81dd0a00dc4f2279da45121f6dec7257622776000000000000000000000000000000000000000000000000000000000000000f0f + 000000000000000002bb: bbd2fb6ed132cf2780a90802acaaa371de119dc89f636dbb647ccdff8b0dc056000000000000000000000000000000000000000000000000000000000000000d0d + 000000000000000002ea: eaa40f6cdd316711961300a0f242fdc226f42d4740c381f05200092eaf70b841000000000000000000000000000000000000000000000000000000000000001212 + 000000000000000002eb: ebdfe46967031db428c3b807c7f8d78f6a51e9ca5f0500ca6099d3d26b1d312a000000000000000000000000000000000000000000000000000000000000001010 + 000000000000000002ef: ef783cc720dbf74c747a155a8975b324d2f8fa80672969dc78fe6f12ea59d03f000000000000000000000000000000000000000000000000000000000000001111 + "00000000000000010205": 05bd0bebbb9c224a78e8d9b6876da7f75cc0d247ce0f6e99f8921fbf410a09a4000000000000000000000000000000000000000000000000000000000000002323 + "00000000000000010207": 07216d95abf0edac40a58f6ca27749af481a4ed3b33dd2d3f10efa814d6da0a7000000000000000000000000000000000000000000000000000000000000001818 + 0000000000000001022d: 2dd4a3d0d6d98198be8e16968d57cda367833b99cfd88c809ce3960ff8b41aba000000000000000000000000000000000000000000000000000000000000002929 + "00000000000000010238": 38ec53adc1cd8bc9f788a5986f73d4e29e2d98945c0aa1d6727be9f8baba4337000000000000000000000000000000000000000000000000000000000000000e0e + 0000000000000001023d: 3d8a8b22175714443f990b996dd26dc71cb53e030d1bf48844df12486e5d4f2a000000000000000000000000000000000000000000000000000000000000002a2a + "00000000000000010249": 49ac53849b70d666cc840b4add0baff56fa9ce7e27be2acb275d109a5994ff8d000000000000000000000000000000000000000000000000000000000000000c0c + 0000000000000001025d: 5daa965d9688be0ac629916a9920e45910c13d1fe45d257a9e17217f226dfb4d000000000000000000000000000000000000000000000000000000000000000101 + 0000000000000001025f: 5f8a83305cccf521595d4bdb1ec8b03be3e9e8e27438db250b9e89f09981799d000000000000000000000000000000000000000000000000000000000000002828 + 0000000000000001026d: 6dfea072e8e999ba0adb48fc1284af16cc1351d53e52b175f9dfe153602b4362000000000000000000000000000000000000000000000000000000000000001414 + "00000000000000010293": 9300ef4007f853d076758bff4c00d6c979113664a6c948d0d313daa379607fdd000000000000000000000000000000000000000000000000000000000000002020 + "00000000000000010299": 99fa0e7a995a9b3c03f9a186d9581dd08387846aade5384b8e85c9c7c193dfb3000000000000000000000000000000000000000000000000000000000000001b1b + 0000000000000001029e: 9e262dd28666dbdb4101e1c27a64deda0e6064360be4c53c89e7b2b35a311943000000000000000000000000000000000000000000000000000000000000002222 + 000000000000000102b3: b314fe2db0b91091038ce12e45838efdfc5c48fe1e68d4f6a4f990ba9e4324f2000000000000000000000000000000000000000000000000000000000000001f1f + 000000000000000102b6: b63eb46438ece57d3d58c40b8193308ffccd3eedec2d645a5ac181f7783aa9c5000000000000000000000000000000000000000000000000000000000000001c1c + 000000000000000102b8: b860ff2535a62373c8c7aa6a31b436cb16427464bde987fc0c35238e10663a56000000000000000000000000000000000000000000000000000000000000002424 + 000000000000000102b9: 002800001632849641ed8fd7f10b2ae3ddaea90de793987e076a54b1d561f0bbc1fa4c65017236b2c6d1e76a12f08ef244830561fd8c3f4cb4eb1d180d9dd3885da41f45cf01 + 000000000000000102dc: dc3e793187abbabede6f6fc80c3ab3ac4a31019d0ef19f59aff10f614a7149fa000000000000000000000000000000000000000000000000000000000000001e1e + 000000000000000102dd: 00280000bf3a7497054eac25ada2fc8c4d315a3a240e6bbe6ecdd3f2fc0f82d8750984410157d9159f07e3d3466d61ab4c26d5b3ea993669fa9b133ee2c9c0a4b859af98b301 + "000000000000000102e8": 0800800005cd913895621437d7b5a41c4cc473ebe257221afa3d6e70297ceacdedfefb2a0137abb18273775dc4a8d30143a3bf7e2db1495570bec8a7876a34f4c18eb9d8eb01 + 000000000000000103b950: b9592813919e07b8df5fd836d1c6e3ac9a9eb7cb66b5ee8a1f4ee0ff3e0b2508000000000000000000000000000000000000000000000000000000000000001919 + 000000000000000103b960: b96e7e15bcbf96c67b1f26fa5ba80089388fbefad39968132c0791cb313d0157000000000000000000000000000000000000000000000000000000000000000404 + 000000000000000103dd50: dd55470824e0db2b94ea10ae29afd473f265bbe758854354b926846821fef91a000000000000000000000000000000000000000000000000000000000000000606 + 000000000000000103dd60: dd69263d904a01d711be57a6f4177bbf6745084517b2ac16468ab075cb88a962000000000000000000000000000000000000000000000000000000000000002727 + "000000000000000103e810": e81cc9e81f01ea7314bf83dc9b7fd942974427fe130fe80b3fb11eed3f80d4ed000000000000000000000000000000000000000000000000000000000000002525 + 000000000000000103e8b0: e8b8b981f358516ba6e7b76e0007bdecf3e97873abe468fbef40110d8206c8d8000000000000000000000000000000000000000000000000000000000000000505 + "00000000000000020201": 01ad1cfd47cdacf2f23714b10102071ea05f91e25ffcf10ac043964bb004b83f000000000000000000000000000000000000000000000000000000000000002c2c + "00000000000000020203": 03484fcec5bd9ccd1b448f7839fe71c8fb7d0e9e9f9076447daa552c89691f60000000000000000000000000000000000000000000000000000000000000003636 + "00000000000000020212": 121a1293f56198bb1639951b344a3ae26f5bca796569bb07d53ce98f4085bcbc000000000000000000000000000000000000000000000000000000000000000808 + "00000000000000020213": 1303216ea6a1a1b396f5fe43017e9869938a896901ed852d95079a1a7d7d57b5000000000000000000000000000000000000000000000000000000000000003434 + "00000000000000020219": 193fbc2a619e98d253554723ff80e1c2de926a8547f09ee1568e53db2b0fa0a2000000000000000000000000000000000000000000000000000000000000003535 + 0000000000000002022e: 2e54be9fbeefab49441556d63666897e33eedb3ff99ed9bffe4b88f9fd85ba11000000000000000000000000000000000000000000000000000000000000003f3f + "00000000000000020252": 5245608ce703d4a307b8c83f268dc60ac248dc26a891baa6116feff1e3cd8ba4000000000000000000000000000000000000000000000000000000000000003939 + "00000000000000020259": 5970e9fe810ebec7d53c18e51fe72b464e75851550b0c34a5e7ebabed8e6c7cc000000000000000000000000000000000000000000000000000000000000002d2d + "00000000000000020260": 00008800b450bcb1628ab1ce5b83630fce3da51e983f02c587d8eba576f264288f799bd2025bf094efc1e3fb61fa5a849f862e091e65738a95d239372e92191d2f5779752b02 + "00000000000000020262": 620975a9aee0d240d9f52ead3ff4f04b9043e260d67bb961fc16188cf1f63635000000000000000000000000000000000000000000000000000000000000003a3a + 0000000000000002029b: 9b420dfe14cdceb77e4464b2a841b6cf6221a29acf6bf363a556a13b72467404000000000000000000000000000000000000000000000000000000000000003d3d + 000000000000000202a0: a0aab27f895e1a504ee0862d8a90c2df63bcc4311409bf357a09d6ba4311d0f8000000000000000000000000000000000000000000000000000000000000003333 + 000000000000000202bd: bd11589aa8ca58a6c00ed1b1354d30b53279703644bed0a73e63513ad11fa5bf000000000000000000000000000000000000000000000000000000000000003737 + 000000000000000202c0: c0519bcddf9b31ea9fbdfe61998e503dca06026d059b990948d1e076657a191e000000000000000000000000000000000000000000000000000000000000002f2f + 000000000000000202c4: c4fbca1f402303e4bc0c85ed3817bc2ff549f665047388b1b40fab23553b2a9d000000000000000000000000000000000000000000000000000000000000001717 + 000000000000000202c7: c78049d38618be10de663e1ef60a5a34a9e635692bf9952c3afb837af3f66e1d000000000000000000000000000000000000000000000000000000000000003232 + 000000000000000202d9: d9898cd6484f797318b72190d1939771527071cac4c2e9da4e053e2b6e0dfe1a000000000000000000000000000000000000000000000000000000000000002b2b + "000000000000000202e2": e2adc6f93c6726bb6f0f56324add68457a28b691aa6e08951cab65ec6f37b54b000000000000000000000000000000000000000000000000000000000000003c3c + 000000000000000202f6: 0082000048902a08b0d7edf4969a608d61ec9911074fc5efb163bc207048d35be9ac0503021993dcbaa58bdd07fccd22a725289a2ab266ee265a3c0dc0eda3a48a154235a102 + "0000000000000002032560": 25659d69d133a28e77d07372508d9bebf0784d709c5f3bfaf1a4d53e22d4febb000000000000000000000000000000000000000000000000000000000000003838 + "0000000000000002032580": 2585f6a0139e0a71940d088e01221a4a5a6852220a894f135cf5c415c511f247000000000000000000000000000000000000000000000000000000000000003030 + "0000000000000002036090": 609f2dcc908d758e6b7dfb10333930babc58eb6ea82a7fb7e332371ba173d855000000000000000000000000000000000000000000000000000000000000002121 + 00000000000000020360b0: 60b22e6aff3665a5256eb5974025b20ba28c2b93ef016a66812f4c6e0c8ef5c4000000000000000000000000000000000000000000000000000000000000002e2e + 000000000000000203d8d0: d8d8c592cc9c74820b9ed9a20fb5709a0d9273f210fbfab39feb94c62f00d806000000000000000000000000000000000000000000000000000000000000003b3b + 000000000000000203d8e0: d8ecaa095dc3b2f77c057c770718c0affe6a9caa0104136143437cf30fb5688e000000000000000000000000000000000000000000000000000000000000003131 + 000000000000000203f640: f6479d69073315ce4df8b1234bd36fde45581f7b61749ba87374c8c71f0b0d72000000000000000000000000000000000000000000000000000000000000001616 + 000000000000000203f670: f673a7125608b9951ba643a43a2b7fc3a87af522e820cccfa65f977949a98c12000000000000000000000000000000000000000000000000000000000000003e3e + "00000000000000030100": 8a8a882257b5edb42fb6c1d47b66df6f451d1f0b07c707b265b47a0a445a49121bf17f09003564f8fd0ce402f3344ebabc39f73df9f5c2c5e7a6ef60d3c65bd3fa126e386b02663d14857c3a5444a050d9655e3827c4728e80d79d1292fd51f52c34b1e7457b0214c2aaac166403626fff92472cec22712462ccd14ca9d431b2cdcc75fe0912cd0026e57d3219488c374ae9bd6c9778ffd012f013335e68a58e614ab7b397b6b3db01941121db179d2296e2f1633f191f7f52dbb69956adb74e105be899b9ab90771c01eb7ced5baec3075725903e2e21b0e1457be9ff668e5e6d40e843117004e4acfa03ae37c191db8c160b309ddf0f969c234bb77fcdf25dfde6f541e0ddffbcfe0de2036f21fa6ab54c46ae0467b9a707bc397efcc6b392f748ee0417583a5a1eb326d6032c9ae2b72aadaed919d02086f2daedba2939863499c8c7a63e1eef851f57dc9900 + "00000000000000030130": 00000a28803101a64d85ff96891a1929b5410e7a7a0b6a60ff8bf3378008164e6c21ae82011d8a35b505566077f9e13da0d84b530d3092ab261a2d2abee46f7f28d74c8e20037d37b6b37acd4cf4045a574c36ae84c9a1dd5f69f307c91f0e72fb20bf0d4fa8013c4e03f48d247afced2623d420be80ad04a1a9e87e27d40dde7cea4f96a9a71803 + "00000000000000030180": 89069ac4c9e498cf8311cdc94afb4afac57857c4d11ff4cfff721be3f2f627b2000000000000000000000000000000000000000000000000000000000000004d4d + 000000000000000301a0: 020882086b44da2a9dd1eaf1e18dad37d7e76005d4b035d5c928629d180c9e0d15554bfb02182fc339100ba14fc2bcbb0ae880c957f3ab0d069eb5f206dd4649004c3fb63100adf6e932d9d77affa57a5662c9976af696c390b173f13f9accdb45d2b3245898030d19ead21c451a82ac4ff7cd7b2f004013811ced4ca96177379ebf5ccc58352900ce290b979ca575bdc45250d691ce3ed7014ec2e83477608208bb75cdead7734903 + 000000000000000301b0: 8224860824620206f5ae3438e207a4bac199dfceedf205d17a5a7f5396b7ceef9eafd0a0032c0a644c823c91a5bea9558492e3b48fd18be253175f64636dbcc860eeee5865017213670135f5ec0a5dfe384bc455d0d377e1c023b39bb0dd1e27e2caaa49a81803c7d6f9eacac49c7dd8949482043c0c42aa0cc5450e38c51e623ed0327ebf094101f3a8e759576757a9c487914a62da083898a037e4aeaa1944c08fe9ff0cfe6a1601409271ef93d4db4c3128801b7e9ae6d181099ca7b8a849670cdc5106b8dad4b801332f352a2bf593288dda47c506db7903535e1900681d84c1bd7cc0ff2fd1548b0034638675b595022e38b43d20ae971594edf5e202eeea2a58bd30f692b2450e1502 + "00000000000000030209": 0959c7836901cba04c833b419ea0dfe0c205bb32a67214c6445b61d7b4a0639b000000000000000000000000000000000000000000000000000000000000004a4a + 0000000000000003020b: 0bebf05465e75ecbdaadb3e79813a435c75b125b2679361a673af1f16b0b8ac8000000000000000000000000000000000000000000000000000000000000004040 + 0000000000000003020c: 0cbf607275c4dca4df0f7ffbb643193ab7a9d7419c0f02e0c3ba79f398f81d46000000000000000000000000000000000000000000000000000000000000004c4c + "00000000000000030239": 39df927e1d5265409744353547dab1f8a27444b2fb917ee896983e046ba91163000000000000000000000000000000000000000000000000000000000000004f4f + 0000000000000003023e: 3e5f86ed85e39b54418692f15640a6bcaf76b7cb6f7b6548da28177df9232ec8000000000000000000000000000000000000000000000000000000000000005252 + "00000000000000030269": 69d819309016b97574c9f345296c54f1432e1a4fc9e48fcc3d3bc03bf4997fda000000000000000000000000000000000000000000000000000000000000004747 + "00000000000000030277": 7783908d59c75dfbcf114da6847cb8abd3be94593bb4267a9b1ebfb223b98ea5000000000000000000000000000000000000000000000000000000000000004b4b + "00000000000000030278": 78421e310f5e842ec956e626534d27008b85f06736ae1f67a1aaa7ed7f21c68f000000000000000000000000000000000000000000000000000000000000005353 + "00000000000000030294": 80000008170ede3e1920837d4865c7c6087bb6117d6083871d773aef1916f8bf1662faea03127afe9fa81b7edde8ad3c48f164abaf9de3ea3c04b5e7eec32bc2609a803c6903 + 0000000000000003029a: 9a559085a559a2e90b54fb02fdd4fd74ab5d604e41c4888c170cdc87c296408e000000000000000000000000000000000000000000000000000000000000004343 + 0000000000000003029f: 9f654242a774697de6c44df46fa9670371ac5999c0b14874dafefb8f4d60638a000000000000000000000000000000000000000000000000000000000000005454 + 000000000000000302a8: a8c56ac32ee75c657919ae3d71f58d15ac0eb2e9d2ff7b9c04b8fd0f96ed7448000000000000000000000000000000000000000000000000000000000000004949 + 000000000000000302ad: ad72cdcdcfb4f0632d055d5445137abf73f21c0dfac21add822c71e82bf2353e000000000000000000000000000000000000000000000000000000000000005151 + 000000000000000302b0: b0e5ace49939a244a59f3654a48da2d9b1c9eb3977925c61a3bac41bfdcfb4ab000000000000000000000000000000000000000000000000000000000000004141 + 000000000000000302b5: 00080200e421f147ed76b69035b1dc7a21a4896daf523abd3a05594f0b9ebe63488ee5700377ab4544f0f4d2663fcc4619ce5a1dd6020cb5f2a4751df29fbb57892276342203 + 000000000000000302c3: c3a199feeed885aebcd589011ee28fdab9550fa9fe53320e56803a6cd4aa0aab000000000000000000000000000000000000000000000000000000000000004242 + 000000000000000302c5: c54f0faa1e221486827abe3f900feb05956f20fc7f15f4e084bd673c489aada4000000000000000000000000000000000000000000000000000000000000004e4e + 000000000000000302d1: d1e5b3c0fc106542ae1485cb8b21e43c3c13630a6a3c20f8615c4b7cdb572490000000000000000000000000000000000000000000000000000000000000004444 + 000000000000000302d8: 000000a821be1d978695211e8e63aa6efccbdf85110a0879cc45f718ddbef1d8392221f80205d8b58aa36651d72ad489469fee23b99bdeab996c96f9aaa1fb691e52b0bdbe029e3d820be42fcb2ad42a93a18226f5eef9ae386b6e926111d97489670383cea103 + "0000000000000003039430": 943a91e688714abcd78bec435e5d1f6412e5132eb61ab16fb71947c26d6dff64000000000000000000000000000000000000000000000000000000000000005050 + 00000000000000030394d0: 94d67b778ebe500b3516f7d892d44cb281067e662cc1483d559cd0d73e00f177000000000000000000000000000000000000000000000000000000000000001d1d + 000000000000000303b550: b553466aa2101212ae2cd93d024063d8367439fdba7b959b730db13a4377f992000000000000000000000000000000000000000000000000000000000000004545 + 000000000000000303b580: b58afc68cab79b077af15ab3b543a110771b218b2c3dd3dc097812e86a89629c000000000000000000000000000000000000000000000000000000000000004848 + 000000000000000303d8f0: d8f4c0f3150c5cc8885c2c0aecbc6819e6689bf4338c70da4816d4cb66ed94f5000000000000000000000000000000000000000000000000000000000000004646 + "000000000000000400": 6455555655c0a8dcf08966f67ecba39300a8722ba44482d5e0b23380a9f213e8c337f590cc03227b208a9a19221548e782068f0fdb6fbd4617116bb152b668c3e9a674ea29c404b49d8c482b50bf07df57e2cc5268959d80fcb3f2da9f0f14d21532f57241f3170453b19451c176ee59d8e6f3c12b6adbd31add13bdea3cec23766e64033aaf8f4d037fa6517eea208326fd635d9ea0a7f08f465d46b631a275df33d13c95e4bf02c504a96003c38bcdb41045b60ec68b377fcd9be9095d5864500f657183cfe777d1e8046dec6c64316471d87e7d08b832fae1293c5bd921e6b05fa6fccc65e0b959119904c062fd7630074d60ddcf9cdfdabdb10356fa4383f98d61c9fdf92909bcfcb819048cedd6f68a9ef7189ace7c6d176c54bd862999e67e1b753e6f91050c255082790357aa7cbdb78645f6e96d2f01c675433421af4d024b10de397ed0c1c85b6cfe030492d380598feb12fe1a1d058ddf0866fe3b7a3ad8fa099914132421d807403033035cd30423af28fd201e24c41618d281a3d0cab380970b73da816a8fbe03117c16035ec4b7a0f7cc597669ca4a4d8d7419d400afd56116b898807f03cdc2ef70bf4e04df49b5b8b47705368b4d5e009e292ec1c92b10ef50501bbb191072bdd0350f3e04c4e5b97afa0931090e318bfaebecf8e2a337476e1a8c102dd970969c6688792f04ba8bc88682602128060614528f82d9a2cae1524e615f6a42cf99a99fec431e4504 + "00000000000000040110": a2000800424573be7b442152cfeb1c3601f99f3130c6f17c7c08b7aaa330c6d87a12515a045c82fb7448102c04e67a1975867232ec52a388958eb4e9e6baeee14c5293abe502503fe6ab38d5ac1c8b7bfe8b097205de90a84730a84de1738a85051d4a4a026302da7578e0017433b8084636a698ea93c593baedaf117de2b36e72020115414bbe02 + "00000000000000040120": 00048828f9c73b9300e34ac8e315a7bff566e806e513bcf859f73fed008bbc294cbd861704fa858129a21ac49dc254a945b31dfe16d6aa38f787508a37e17770d646efc8440054811a6dbdccae4f94ec1115b47b52a93363dbfaa0997714af738a41b673af93008f46a18e5863c4ada0e7cd39134e090bc980515a25e481ffc3763ff91c26e91401fdcebb4312eac9d2281efff9ac70c4693fe3a7e5d1be16a233845059ce1d9e3e02 + "00000000000000040140": 00820900ac90a14f9020bf1306cce40cb69603c44f376b19b3b89b164da4adad2522f3cd0444b03ea062f4472e00e34d5e0293dd01552dc38a7a87408fd085c93074aa6c4104a71618826a808ad48f199fd524a76e7dd86dbebc0f41833f7831fbc6841282d004af4063b99148e997961d12ee3dd264572aeb229046a8e5e9b225dc1dfacc72fc01 + "00000000000000040150": 202008a8c9994c2caa7d57098169757caf7127daad05bbc20a0f2482c66e0bd3e0f2425802351e6086b7c92ade4db538680c6d267f479fccb2d4ce9b10062afdd17369911f04e6d61bcc44717172e1e2c6b2ff387448722c2fc17a17726189a0c93514e5dab4029fa3dba19b7c08af62edc9239b652fb5675abe268ec732eb467087969ba6084b01a64d06efc9a752b9453f768fd58528f039f4fc63aff3e8353f3268ecf15df977046fb85f4290b650931131cbbd4f910b772185cd8006f0b917801678a34b4fa79101 + "00000000000000040160": 2100880851a6e73460f30029943bf553f273b1a48ccd31437e3f83a25fb38dffb2d6ac28020724c3ee74e7f23cb76e9395e87d1e4bff63f1da7049a6eca441724797f3874b02cf2fe9fd420120d82d0609329ca9b5ffa91da307e1b7f51057b747c639a34af8034340fb29a51a13adbfe03a805124b70da17033d7707cfbb0dfcd54d24c1e63d404aaadd43d4d5793871733182fe8506eed15062b5875e420c8b7d22cba53ac378501 + "00000000000000040170": 0080822205a922b3f8195dca99196eacd675b9eb833f50040232bb1f618834baf5b38ba103167a1e11488fdfa9d5b12121dd9910adfbf852c209f3a577b967000745f350f603aacca0ce4f75eee32589c24eec3958eeccef822af3b15daaa432fe3e82a05cb804dab1eee1892b8092343e6487bc31fa80fb2dd7cbb9ed1c6fae878f83dded1c2d00060b08b1a2e058cb5ec3f70f413c5d5440ef7d6e542223d3784fbd6c72e8c68c00 + "00000000000000040190": 8005a8a07dd0df391d962fd4150f793cbc18c888b63ed02cf54a3d0db5cb3f2b29c36e24019fbced1659f7cd1a5c0af9ee189842e1692e65352c1b487f61f2d752f2a0b3d70359773d2c25f7cb39674f6ae2cbc3fbdd82d1ed6972a6aa3ed016269f3e55502404c817723a3d92295eccbaf6eba511b46779dbe67626cde023cf8cd37f33460654018ca3d9745931ef68100145097c22269521f4df6a11f5b2d3e3488216298a8a7603b99a204e2e9faa22aaf21289c9e4f0495f1a388969a5172bc620b50bfa30ddab026f5a9ea74107c8f4de693c261104144e5bc30d5c84fd24817d46e2335b58336701e16772b9714e64a1009a37324bbe60a52cbe8674e7964b8cf9982e83b33ff64103 + 000000000000000401c0: 828a2000c44c8b773c992751a9a68cf562a5d66a82ecb3e5b1f3e825583af4a4dc7188fb02f8e73c81b1ec28c666c9a504eb2be2aa8a2cc1cf0af2889447eea287118568bf032de02cba921be88fba820ac33a520ad265be4c4a290d864f0377884abf900ac002e8927e187971ca77095814caa6cc732835445a638edb782610240ae86388c788030ba1131ae7242bf1d1dfc8c0f32eb0a5cf80c6f85dab3d0d0c5355881ead0e86022e3b99b6d3c2022157528163ef99f3fb81204f8d3f32ff3bcd8fb918e409190904 + 000000000000000401d0: 08800906f4063981ea97ecdb06015df9bf4f6727c57efbf4b0646c3476deb36794324db103afd85ebff302f5bbba6058397c4ea0828d22a3778df3f112311c1d0accf7c708042b1948ebf5edfef56db3a187d77bc30e4b3200612376c93a909b907ff574c6e5037eae10ad154a3d3f012dcfce2fe0adf14890f879676eb80b621029c68d316ca302d82817436305281ec13d95108dab7f12d2efe6baeef0e20c192522e3fb9acfd20190931c85595b1e71a6693a36c92ac5c7859148c77e098b7c950389ab74c5cfcb01 + "000000000000000401e0": 2880a18043fe9188b8b985568354bcdd9250b431c4e714f6b42051be284857a8d2be1722045a0aecb3ba406c7d3e437d8ef8e625e2d77e062da840461ccf48c0c4308f403402299f131a3be0248d2e798e4ad2f600e15e0f33a6cd627aded2521f4ec15f997804f11b8ad362ee769da41a1cd9c52ad888b677eec1f20671001f8777faa9cf964a01fc7d7cf7cf7a7beefa4a24a16c1aee39a859559a0af0057b8ee2d08190b6f92000ac0b59ebc0d792db7fc244e10d924fca664b6f6e2ceec70264212ddc795c962f0012c7eac2be7db3066f6ac802ba777e9eb44c7a5cab845977367ad8b34df71bc500 + 000000000000000401f0: 881000005ee18ade69f359305864cf1d17d37a06c42736b877fafc52e64c3413655b96b104aaa833785c900c5124bf8ed8d919ee7eaa067d54c92198804d10e1cd19bffbc104815452859f52a833bb33a0648e223514255bf86b3b57a022a2cc9e761ce5b5d402 + "00000000000000040210": 10517f8d773163c41a3ad0446b196c056622643019f2f0f7b6ee0bd6374e224e000000000000000000000000000000000000000000000000000000000000005e5e + "00000000000000040225": 0022020063a7ef2de46a7ccd3a1fa06a407e928949b80e72ddb6b3db95ec8715acd6f7f70475611c7ae282df1b3ad74bb8b08fc01ee7f4dd46036cf91eb514376122e4a62302592ab1d5500c2ed989fa665f501c67499af68a98a0090ea4c7e66cf53684fc0c02 + "00000000000000040244": 446c0030d10b7a6b6c2970f02f4505d71de7bbf95c0c99884c0c01467f53a8ab000000000000000000000000000000000000000000000000000000000000005858 + "00000000000000040247": 473c0e43a2c5fd95e1b6b6154e7a720a5d8d257790f508c601633aff2bd9e48d000000000000000000000000000000000000000000000000000000000000005f5f + "00000000000000040248": 0008002091380386034a68521194d26b451f80c900eed4fb13cb9e8913e3dfc5e592b00a0452d96f4eeda24e81dd90939b2748dacd8a765a938bde5574cc54544aa00d194d04 + "00000000000000040256": 56caf919aa7245d39566bd66e00d1a91b9894333d5d0384b6a2a12c6e66d95d1000000000000000000000000000000000000000000000000000000000000006161 + 0000000000000004025e: 5e3c1c034ab4b4eead015b30d1a272db741bb510d9e3d3f1abd099652302414f000000000000000000000000000000000000000000000000000000000000005c5c + 0000000000000004026b: 6b0fd3cfa6040f65f7276c040ccc7a4b246f90de0d99b1d3d43a2feac6c894bd000000000000000000000000000000000000000000000000000000000000006464 + 0000000000000004027b: 7b26af81f4c1ab7cdaa6fc0c4aec935004631a29ea1382ac3827df603e42a9db000000000000000000000000000000000000000000000000000000000000005b5b + "00000000000000040295": 002000086e520d01c9c38a65012be48c7d7a1ee5de3f33497cd4de2605f2998d6f8c0398045c96cbd1a7d9f8be9e808c5e59d525622725bc800f5ca3aeb5e06ef58f96b05c04 + 000000000000000402ca: ca19a24e1132928718d987a578a4ca038eab9602e8c4abf553bd7189459effef000000000000000000000000000000000000000000000000000000000000005656 + 000000000000000402d7: d753f729b5ccfe040f4f592b95b710c4e6f147bb9178880080f2d094becfad86000000000000000000000000000000000000000000000000000000000000005a5a + "000000000000000402e1": e13b9b67638cdad23dec7259168f9562d1dc889b2be70d752bc3dfe82bc6a712000000000000000000000000000000000000000000000000000000000000005757 + "000000000000000402e7": e7df81ef1c550b915e5e9060719aa92ac78a1fddd47c325b79435a7752f7e5b7000000000000000000000000000000000000000000000000000000000000005959 + 000000000000000402f1: f1a7fba0a3a8c6f5ea1249d0c556b27da90c998fe9d6dc565911b5638f66f5c4000000000000000000000000000000000000000000000000000000000000006363 + 000000000000000402f3: f38ccb0299ceb194f40be88e74d175c707832dc74a73b2087aa1027689cd40f4000000000000000000000000000000000000000000000000000000000000006060 + "0000000000000004032540": 25447bdfd63dd3e622671ad29dd6360a085ce6006ac74a6c0d6a3c279ffc3097000000000000000000000000000000000000000000000000000000000000005d5d + "0000000000000004034850": 4858ed2508d9a8a36c8c771fd9cdd71285d5c4dc624a1e63c0abd64cef699b03000000000000000000000000000000000000000000000000000000000000002626 + "00000000000000040348e0": 48e2e8edc61fc5bcddff86b114b5a870369db85287fe39ee8b9a63837552c315000000000000000000000000000000000000000000000000000000000000005555 + "0000000000000004039560": 956ba33c478bde6c11ea2a1f6105753d55271cad5ce7101a08b8547a234e74c6000000000000000000000000000000000000000000000000000000000000001a1a + 00000000000000040395d0: 95dbef93d0be459e34c6e6663551ebc31befb871daa6430a62329531bde4add7000000000000000000000000000000000000000000000000000000000000006262 +stale_keys: {} + diff --git a/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-3-chunked-commits-pruned.snap b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-3-chunked-commits-pruned.snap new file mode 100644 index 000000000000..608e61167cf3 --- /dev/null +++ b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-3-chunked-commits-pruned.snap @@ -0,0 +1,136 @@ +--- +source: core/lib/merkle_tree/tests/integration/merkle_tree.rs +assertion_line: 658 +expression: "DatabaseSnapshot::new(&raw_db)" +--- +tree: + "00": 22030c61726368697465637475726506415231364d5405646570746803323536066861736865720a626c616b653273323536 + "00000000000000020200": 0059914596ed2e70745c44ef315747ea29aff72bce6757aca30a434d9bb70781000000000000000000000000000000000000000000000000000000000000000202 + "00000000000000020204": 049dfddb9f03237fed8d902e350f0b0e9b85af93f49fe02f0d189c47cd7b122b000000000000000000000000000000000000000000000000000000000000000707 + 0000000000000002027c: 7cabd5f57ff72670052939485c4533b01b26fd0e490c31bd48b5ac5002ff1f83000000000000000000000000000000000000000000000000000000000000000909 + 0000000000000002027e: 7ed4dea78574266e019059e5b5fd6f94ed1632bd4a643d1c51aa02974def5684000000000000000000000000000000000000000000000000000000000000000303 + 0000000000000003020e: 0edeb726b4588b4b373d38b7b43e3083c6e105c991e1812bdcce6c373ed98dd5000000000000000000000000000000000000000000000000000000000000000a0a + 000000000000000402bb: bbd2fb6ed132cf2780a90802acaaa371de119dc89f636dbb647ccdff8b0dc056000000000000000000000000000000000000000000000000000000000000000d0d + 000000000000000502ea: eaa40f6cdd316711961300a0f242fdc226f42d4740c381f05200092eaf70b841000000000000000000000000000000000000000000000000000000000000001212 + 000000000000000502eb: ebdfe46967031db428c3b807c7f8d78f6a51e9ca5f0500ca6099d3d26b1d312a000000000000000000000000000000000000000000000000000000000000001010 + 000000000000000502ef: ef783cc720dbf74c747a155a8975b324d2f8fa80672969dc78fe6f12ea59d03f000000000000000000000000000000000000000000000000000000000000001111 + "00000000000000060229": 297faa5f83989e2a8b574589c3eeb5fdced0634fd75d662507604796dd4edd1b000000000000000000000000000000000000000000000000000000000000001515 + 0000000000000006022b: 2b5ffd54bff018009f93f7cd782e8922b24520eb99a100efe2c1a4ae6c1ca3f5000000000000000000000000000000000000000000000000000000000000000b0b + 000000000000000602a5: a58e3a77937b9b747f70d4aee9e84992d7955e52e2de71dc9615df3d16b2b816000000000000000000000000000000000000000000000000000000000000001313 + 000000000000000602ab: ab314b8d202e718d011f9f90df81dd0a00dc4f2279da45121f6dec7257622776000000000000000000000000000000000000000000000000000000000000000f0f + "00000000000000070207": 07216d95abf0edac40a58f6ca27749af481a4ed3b33dd2d3f10efa814d6da0a7000000000000000000000000000000000000000000000000000000000000001818 + "00000000000000080299": 99fa0e7a995a9b3c03f9a186d9581dd08387846aade5384b8e85c9c7c193dfb3000000000000000000000000000000000000000000000000000000000000001b1b + 000000000000000802b9: 002800001632849641ed8fd7f10b2ae3ddaea90de793987e076a54b1d561f0bbc1fa4c65087236b2c6d1e76a12f08ef244830561fd8c3f4cb4eb1d180d9dd3885da41f45cf08 + 000000000000000803b950: b9592813919e07b8df5fd836d1c6e3ac9a9eb7cb66b5ee8a1f4ee0ff3e0b2508000000000000000000000000000000000000000000000000000000000000001919 + 000000000000000803b960: b96e7e15bcbf96c67b1f26fa5ba80089388fbefad39968132c0791cb313d0157000000000000000000000000000000000000000000000000000000000000000404 + 000000000000000902b6: b63eb46438ece57d3d58c40b8193308ffccd3eedec2d645a5ac181f7783aa9c5000000000000000000000000000000000000000000000000000000000000001c1c + 000000000000000902dc: dc3e793187abbabede6f6fc80c3ab3ac4a31019d0ef19f59aff10f614a7149fa000000000000000000000000000000000000000000000000000000000000001e1e + 000000000000000a026d: 6dfea072e8e999ba0adb48fc1284af16cc1351d53e52b175f9dfe153602b4362000000000000000000000000000000000000000000000000000000000000001414 + 000000000000000a0293: 9300ef4007f853d076758bff4c00d6c979113664a6c948d0d313daa379607fdd000000000000000000000000000000000000000000000000000000000000002020 + 000000000000000a02b3: b314fe2db0b91091038ce12e45838efdfc5c48fe1e68d4f6a4f990ba9e4324f2000000000000000000000000000000000000000000000000000000000000001f1f + 000000000000000b0205: 05bd0bebbb9c224a78e8d9b6876da7f75cc0d247ce0f6e99f8921fbf410a09a4000000000000000000000000000000000000000000000000000000000000002323 + 000000000000000b029e: 9e262dd28666dbdb4101e1c27a64deda0e6064360be4c53c89e7b2b35a311943000000000000000000000000000000000000000000000000000000000000002222 + 000000000000000b02b8: b860ff2535a62373c8c7aa6a31b436cb16427464bde987fc0c35238e10663a56000000000000000000000000000000000000000000000000000000000000002424 + 000000000000000c0249: 49ac53849b70d666cc840b4add0baff56fa9ce7e27be2acb275d109a5994ff8d000000000000000000000000000000000000000000000000000000000000000c0c + 000000000000000c02dd: 00280000bf3a7497054eac25ada2fc8c4d315a3a240e6bbe6ecdd3f2fc0f82d8750984410c57d9159f07e3d3466d61ab4c26d5b3ea993669fa9b133ee2c9c0a4b859af98b30c + 000000000000000c02e8: 0800800005cd913895621437d7b5a41c4cc473ebe257221afa3d6e70297ceacdedfefb2a0c37abb18273775dc4a8d30143a3bf7e2db1495570bec8a7876a34f4c18eb9d8eb0c + 000000000000000c03dd50: dd55470824e0db2b94ea10ae29afd473f265bbe758854354b926846821fef91a000000000000000000000000000000000000000000000000000000000000000606 + 000000000000000c03dd60: dd69263d904a01d711be57a6f4177bbf6745084517b2ac16468ab075cb88a962000000000000000000000000000000000000000000000000000000000000002727 + 000000000000000c03e810: e81cc9e81f01ea7314bf83dc9b7fd942974427fe130fe80b3fb11eed3f80d4ed000000000000000000000000000000000000000000000000000000000000002525 + 000000000000000c03e8b0: e8b8b981f358516ba6e7b76e0007bdecf3e97873abe468fbef40110d8206c8d8000000000000000000000000000000000000000000000000000000000000000505 + 000000000000000d022d: 2dd4a3d0d6d98198be8e16968d57cda367833b99cfd88c809ce3960ff8b41aba000000000000000000000000000000000000000000000000000000000000002929 + 000000000000000d0238: 38ec53adc1cd8bc9f788a5986f73d4e29e2d98945c0aa1d6727be9f8baba4337000000000000000000000000000000000000000000000000000000000000000e0e + 000000000000000d023d: 3d8a8b22175714443f990b996dd26dc71cb53e030d1bf48844df12486e5d4f2a000000000000000000000000000000000000000000000000000000000000002a2a + 000000000000000d025d: 5daa965d9688be0ac629916a9920e45910c13d1fe45d257a9e17217f226dfb4d000000000000000000000000000000000000000000000000000000000000000101 + 000000000000000d025f: 5f8a83305cccf521595d4bdb1ec8b03be3e9e8e27438db250b9e89f09981799d000000000000000000000000000000000000000000000000000000000000002828 + "000000000000000e0201": 01ad1cfd47cdacf2f23714b10102071ea05f91e25ffcf10ac043964bb004b83f000000000000000000000000000000000000000000000000000000000000002c2c + "000000000000000e0259": 5970e9fe810ebec7d53c18e51fe72b464e75851550b0c34a5e7ebabed8e6c7cc000000000000000000000000000000000000000000000000000000000000002d2d + 000000000000000e02d9: d9898cd6484f797318b72190d1939771527071cac4c2e9da4e053e2b6e0dfe1a000000000000000000000000000000000000000000000000000000000000002b2b + 000000000000000f0260: 00008800b450bcb1628ab1ce5b83630fce3da51e983f02c587d8eba576f264288f799bd20f5bf094efc1e3fb61fa5a849f862e091e65738a95d239372e92191d2f5779752b0f + 000000000000000f02c0: c0519bcddf9b31ea9fbdfe61998e503dca06026d059b990948d1e076657a191e000000000000000000000000000000000000000000000000000000000000002f2f + 000000000000000f02c4: c4fbca1f402303e4bc0c85ed3817bc2ff549f665047388b1b40fab23553b2a9d000000000000000000000000000000000000000000000000000000000000001717 + 000000000000000f036090: 609f2dcc908d758e6b7dfb10333930babc58eb6ea82a7fb7e332371ba173d855000000000000000000000000000000000000000000000000000000000000002121 + 000000000000000f0360b0: 60b22e6aff3665a5256eb5974025b20ba28c2b93ef016a66812f4c6e0c8ef5c4000000000000000000000000000000000000000000000000000000000000002e2e + 000000000000001002a0: a0aab27f895e1a504ee0862d8a90c2df63bcc4311409bf357a09d6ba4311d0f8000000000000000000000000000000000000000000000000000000000000003333 + 000000000000001002c7: c78049d38618be10de663e1ef60a5a34a9e635692bf9952c3afb837af3f66e1d000000000000000000000000000000000000000000000000000000000000003232 + "00000000000000110203": 03484fcec5bd9ccd1b448f7839fe71c8fb7d0e9e9f9076447daa552c89691f60000000000000000000000000000000000000000000000000000000000000003636 + "00000000000000110212": 121a1293f56198bb1639951b344a3ae26f5bca796569bb07d53ce98f4085bcbc000000000000000000000000000000000000000000000000000000000000000808 + "00000000000000110213": 1303216ea6a1a1b396f5fe43017e9869938a896901ed852d95079a1a7d7d57b5000000000000000000000000000000000000000000000000000000000000003434 + "00000000000000110219": 193fbc2a619e98d253554723ff80e1c2de926a8547f09ee1568e53db2b0fa0a2000000000000000000000000000000000000000000000000000000000000003535 + "00000000000000120252": 5245608ce703d4a307b8c83f268dc60ac248dc26a891baa6116feff1e3cd8ba4000000000000000000000000000000000000000000000000000000000000003939 + 000000000000001202bd: bd11589aa8ca58a6c00ed1b1354d30b53279703644bed0a73e63513ad11fa5bf000000000000000000000000000000000000000000000000000000000000003737 + "0000000000000012032560": 25659d69d133a28e77d07372508d9bebf0784d709c5f3bfaf1a4d53e22d4febb000000000000000000000000000000000000000000000000000000000000003838 + "0000000000000012032580": 2585f6a0139e0a71940d088e01221a4a5a6852220a894f135cf5c415c511f247000000000000000000000000000000000000000000000000000000000000003030 + "00000000000000130262": 620975a9aee0d240d9f52ead3ff4f04b9043e260d67bb961fc16188cf1f63635000000000000000000000000000000000000000000000000000000000000003a3a + "000000000000001302e2": e2adc6f93c6726bb6f0f56324add68457a28b691aa6e08951cab65ec6f37b54b000000000000000000000000000000000000000000000000000000000000003c3c + 000000000000001303d8d0: d8d8c592cc9c74820b9ed9a20fb5709a0d9273f210fbfab39feb94c62f00d806000000000000000000000000000000000000000000000000000000000000003b3b + 000000000000001303d8e0: d8ecaa095dc3b2f77c057c770718c0affe6a9caa0104136143437cf30fb5688e000000000000000000000000000000000000000000000000000000000000003131 + 0000000000000014022e: 2e54be9fbeefab49441556d63666897e33eedb3ff99ed9bffe4b88f9fd85ba11000000000000000000000000000000000000000000000000000000000000003f3f + 0000000000000014029b: 9b420dfe14cdceb77e4464b2a841b6cf6221a29acf6bf363a556a13b72467404000000000000000000000000000000000000000000000000000000000000003d3d + 000000000000001402f6: 0082000048902a08b0d7edf4969a608d61ec9911074fc5efb163bc207048d35be9ac0503141993dcbaa58bdd07fccd22a725289a2ab266ee265a3c0dc0eda3a48a154235a114 + 000000000000001403f640: f6479d69073315ce4df8b1234bd36fde45581f7b61749ba87374c8c71f0b0d72000000000000000000000000000000000000000000000000000000000000001616 + 000000000000001403f670: f673a7125608b9951ba643a43a2b7fc3a87af522e820cccfa65f977949a98c12000000000000000000000000000000000000000000000000000000000000003e3e + 0000000000000015020b: 0bebf05465e75ecbdaadb3e79813a435c75b125b2679361a673af1f16b0b8ac8000000000000000000000000000000000000000000000000000000000000004040 + 000000000000001502b0: b0e5ace49939a244a59f3654a48da2d9b1c9eb3977925c61a3bac41bfdcfb4ab000000000000000000000000000000000000000000000000000000000000004141 + 000000000000001502c3: c3a199feeed885aebcd589011ee28fdab9550fa9fe53320e56803a6cd4aa0aab000000000000000000000000000000000000000000000000000000000000004242 + 0000000000000016029a: 9a559085a559a2e90b54fb02fdd4fd74ab5d604e41c4888c170cdc87c296408e000000000000000000000000000000000000000000000000000000000000004343 + 000000000000001602d1: d1e5b3c0fc106542ae1485cb8b21e43c3c13630a6a3c20f8615c4b7cdb572490000000000000000000000000000000000000000000000000000000000000004444 + 000000000000001701b0: 8224860824620206f5ae3438e207a4bac199dfceedf205d17a5a7f5396b7ceef9eafd0a0152c0a644c823c91a5bea9558492e3b48fd18be253175f64636dbcc860eeee58650a7213670135f5ec0a5dfe384bc455d0d377e1c023b39bb0dd1e27e2caaa49a81817c7d6f9eacac49c7dd8949482043c0c42aa0cc5450e38c51e623ed0327ebf094109f3a8e759576757a9c487914a62da083898a037e4aeaa1944c08fe9ff0cfe6a160b409271ef93d4db4c3128801b7e9ae6d181099ca7b8a849670cdc5106b8dad4b808332f352a2bf593288dda47c506db7903535e1900681d84c1bd7cc0ff2fd1548b0434638675b595022e38b43d20ae971594edf5e202eeea2a58bd30f692b2450e1512 + "00000000000000170269": 69d819309016b97574c9f345296c54f1432e1a4fc9e48fcc3d3bc03bf4997fda000000000000000000000000000000000000000000000000000000000000004747 + 000000000000001702b5: 00080200e421f147ed76b69035b1dc7a21a4896daf523abd3a05594f0b9ebe63488ee5701777ab4544f0f4d2663fcc4619ce5a1dd6020cb5f2a4751df29fbb57892276342217 + 000000000000001702d8: 000000a821be1d978695211e8e63aa6efccbdf85110a0879cc45f718ddbef1d8392221f81305d8b58aa36651d72ad489469fee23b99bdeab996c96f9aaa1fb691e52b0bdbe139e3d820be42fcb2ad42a93a18226f5eef9ae386b6e926111d97489670383cea117 + 000000000000001703b550: b553466aa2101212ae2cd93d024063d8367439fdba7b959b730db13a4377f992000000000000000000000000000000000000000000000000000000000000004545 + 000000000000001703b580: b58afc68cab79b077af15ab3b543a110771b218b2c3dd3dc097812e86a89629c000000000000000000000000000000000000000000000000000000000000004848 + 000000000000001703d8f0: d8f4c0f3150c5cc8885c2c0aecbc6819e6689bf4338c70da4816d4cb66ed94f5000000000000000000000000000000000000000000000000000000000000004646 + "00000000000000180209": 0959c7836901cba04c833b419ea0dfe0c205bb32a67214c6445b61d7b4a0639b000000000000000000000000000000000000000000000000000000000000004a4a + "00000000000000180277": 7783908d59c75dfbcf114da6847cb8abd3be94593bb4267a9b1ebfb223b98ea5000000000000000000000000000000000000000000000000000000000000004b4b + 000000000000001802a8: a8c56ac32ee75c657919ae3d71f58d15ac0eb2e9d2ff7b9c04b8fd0f96ed7448000000000000000000000000000000000000000000000000000000000000004949 + "00000000000000190100": 8a8a882257b5edb42fb6c1d47b66df6f451d1f0b07c707b265b47a0a445a49121bf17f09023564f8fd0ce402f3344ebabc39f73df9f5c2c5e7a6ef60d3c65bd3fa126e386b0e663d14857c3a5444a050d9655e3827c4728e80d79d1292fd51f52c34b1e7457b1114c2aaac166403626fff92472cec22712462ccd14ca9d431b2cdcc75fe0912cd0226e57d3219488c374ae9bd6c9778ffd012f013335e68a58e614ab7b397b6b3db0b941121db179d2296e2f1633f191f7f52dbb69956adb74e105be899b9ab90771c07eb7ced5baec3075725903e2e21b0e1457be9ff668e5e6d40e843117004e4acfa18ae37c191db8c160b309ddf0f969c234bb77fcdf25dfde6f541e0ddffbcfe0de2156f21fa6ab54c46ae0467b9a707bc397efcc6b392f748ee0417583a5a1eb326d6192c9ae2b72aadaed919d02086f2daedba2939863499c8c7a63e1eef851f57dc9903 + "00000000000000190180": 89069ac4c9e498cf8311cdc94afb4afac57857c4d11ff4cfff721be3f2f627b2000000000000000000000000000000000000000000000000000000000000004d4d + 0000000000000019020c: 0cbf607275c4dca4df0f7ffbb643193ab7a9d7419c0f02e0c3ba79f398f81d46000000000000000000000000000000000000000000000000000000000000004c4c + 000000000000001902c5: c54f0faa1e221486827abe3f900feb05956f20fc7f15f4e084bd673c489aada4000000000000000000000000000000000000000000000000000000000000004e4e + 000000000000001a01a0: 020882086b44da2a9dd1eaf1e18dad37d7e76005d4b035d5c928629d180c9e0d15554bfb10182fc339100ba14fc2bcbb0ae880c957f3ab0d069eb5f206dd4649004c3fb63106adf6e932d9d77affa57a5662c9976af696c390b173f13f9accdb45d2b3245898180d19ead21c451a82ac4ff7cd7b2f004013811ced4ca96177379ebf5ccc58352906ce290b979ca575bdc45250d691ce3ed7014ec2e83477608208bb75cdead773491a + 000000000000001a0239: 39df927e1d5265409744353547dab1f8a27444b2fb917ee896983e046ba91163000000000000000000000000000000000000000000000000000000000000004f4f + 000000000000001a0294: 80000008170ede3e1920837d4865c7c6087bb6117d6083871d773aef1916f8bf1662faea1a127afe9fa81b7edde8ad3c48f164abaf9de3ea3c04b5e7eec32bc2609a803c691a + 000000000000001a02ad: ad72cdcdcfb4f0632d055d5445137abf73f21c0dfac21add822c71e82bf2353e000000000000000000000000000000000000000000000000000000000000005151 + 000000000000001a039430: 943a91e688714abcd78bec435e5d1f6412e5132eb61ab16fb71947c26d6dff64000000000000000000000000000000000000000000000000000000000000005050 + 000000000000001a0394d0: 94d67b778ebe500b3516f7d892d44cb281067e662cc1483d559cd0d73e00f177000000000000000000000000000000000000000000000000000000000000001d1d + 000000000000001b0130: 00000a28803101a64d85ff96891a1929b5410e7a7a0b6a60ff8bf3378008164e6c21ae820d1d8a35b505566077f9e13da0d84b530d3092ab261a2d2abee46f7f28d74c8e201a7d37b6b37acd4cf4045a574c36ae84c9a1dd5f69f307c91f0e72fb20bf0d4fa80d3c4e03f48d247afced2623d420be80ad04a1a9e87e27d40dde7cea4f96a9a7181b + 000000000000001b023e: 3e5f86ed85e39b54418692f15640a6bcaf76b7cb6f7b6548da28177df9232ec8000000000000000000000000000000000000000000000000000000000000005252 + 000000000000001b0278: 78421e310f5e842ec956e626534d27008b85f06736ae1f67a1aaa7ed7f21c68f000000000000000000000000000000000000000000000000000000000000005353 + 000000000000001b029f: 9f654242a774697de6c44df46fa9670371ac5999c0b14874dafefb8f4d60638a000000000000000000000000000000000000000000000000000000000000005454 + 000000000000001c01c0: 828a2000c44c8b773c992751a9a68cf562a5d66a82ecb3e5b1f3e825583af4a4dc7188fb0ff8e73c81b1ec28c666c9a504eb2be2aa8a2cc1cf0af2889447eea287118568bf152de02cba921be88fba820ac33a520ad265be4c4a290d864f0377884abf900ac00fe8927e187971ca77095814caa6cc732835445a638edb782610240ae86388c788190ba1131ae7242bf1d1dfc8c0f32eb0a5cf80c6f85dab3d0d0c5355881ead0e86102e3b99b6d3c2022157528163ef99f3fb81204f8d3f32ff3bcd8fb918e40919091c + 000000000000001c0248: 0008002091380386034a68521194d26b451f80c900eed4fb13cb9e8913e3dfc5e592b00a1c52d96f4eeda24e81dd90939b2748dacd8a765a938bde5574cc54544aa00d194d1c + 000000000000001c02ca: ca19a24e1132928718d987a578a4ca038eab9602e8c4abf553bd7189459effef000000000000000000000000000000000000000000000000000000000000005656 + 000000000000001c02e1: e13b9b67638cdad23dec7259168f9562d1dc889b2be70d752bc3dfe82bc6a712000000000000000000000000000000000000000000000000000000000000005757 + 000000000000001c034850: 4858ed2508d9a8a36c8c771fd9cdd71285d5c4dc624a1e63c0abd64cef699b03000000000000000000000000000000000000000000000000000000000000002626 + 000000000000001c0348e0: 48e2e8edc61fc5bcddff86b114b5a870369db85287fe39ee8b9a63837552c315000000000000000000000000000000000000000000000000000000000000005555 + 000000000000001d01d0: 08800906f4063981ea97ecdb06015df9bf4f6727c57efbf4b0646c3476deb36794324db116afd85ebff302f5bbba6058397c4ea0828d22a3778df3f112311c1d0accf7c7081d2b1948ebf5edfef56db3a187d77bc30e4b3200612376c93a909b907ff574c6e5177eae10ad154a3d3f012dcfce2fe0adf14890f879676eb80b621029c68d316ca30ed82817436305281ec13d95108dab7f12d2efe6baeef0e20c192522e3fb9acfd20990931c85595b1e71a6693a36c92ac5c7859148c77e098b7c950389ab74c5cfcb0c + 000000000000001d01e0: 2880a18043fe9188b8b985568354bcdd9250b431c4e714f6b42051be284857a8d2be17221c5a0aecb3ba406c7d3e437d8ef8e625e2d77e062da840461ccf48c0c4308f403413299f131a3be0248d2e798e4ad2f600e15e0f33a6cd627aded2521f4ec15f99781df11b8ad362ee769da41a1cd9c52ad888b677eec1f20671001f8777faa9cf964a0cfc7d7cf7cf7a7beefa4a24a16c1aee39a859559a0af0057b8ee2d08190b6f92005ac0b59ebc0d792db7fc244e10d924fca664b6f6e2ceec70264212ddc795c962f0512c7eac2be7db3066f6ac802ba777e9eb44c7a5cab845977367ad8b34df71bc505 + 000000000000001d0244: 446c0030d10b7a6b6c2970f02f4505d71de7bbf95c0c99884c0c01467f53a8ab000000000000000000000000000000000000000000000000000000000000005858 + 000000000000001d02d7: d753f729b5ccfe040f4f592b95b710c4e6f147bb9178880080f2d094becfad86000000000000000000000000000000000000000000000000000000000000005a5a + 000000000000001d02e7: e7df81ef1c550b915e5e9060719aa92ac78a1fddd47c325b79435a7752f7e5b7000000000000000000000000000000000000000000000000000000000000005959 + "000000000000001e0120": 00048828f9c73b9300e34ac8e315a7bff566e806e513bcf859f73fed008bbc294cbd86171efa858129a21ac49dc254a945b31dfe16d6aa38f787508a37e17770d646efc8440654811a6dbdccae4f94ec1115b47b52a93363dbfaa0997714af738a41b673af93068f46a18e5863c4ada0e7cd39134e090bc980515a25e481ffc3763ff91c26e9140dfdcebb4312eac9d2281efff9ac70c4693fe3a7e5d1be16a233845059ce1d9e3e14 + "000000000000001e0170": 0080822205a922b3f8195dca99196eacd675b9eb833f50040232bb1f618834baf5b38ba118167a1e11488fdfa9d5b12121dd9910adfbf852c209f3a577b967000745f350f61baacca0ce4f75eee32589c24eec3958eeccef822af3b15daaa432fe3e82a05cb81edab1eee1892b8092343e6487bc31fa80fb2dd7cbb9ed1c6fae878f83dded1c2d02060b08b1a2e058cb5ec3f70f413c5d5440ef7d6e542223d3784fbd6c72e8c68c02 + "000000000000001e0225": 0022020063a7ef2de46a7ccd3a1fa06a407e928949b80e72ddb6b3db95ec8715acd6f7f71e75611c7ae282df1b3ad74bb8b08fc01ee7f4dd46036cf91eb514376122e4a62312592ab1d5500c2ed989fa665f501c67499af68a98a0090ea4c7e66cf53684fc0c12 + 000000000000001e025e: 5e3c1c034ab4b4eead015b30d1a272db741bb510d9e3d3f1abd099652302414f000000000000000000000000000000000000000000000000000000000000005c5c + 000000000000001e027b: 7b26af81f4c1ab7cdaa6fc0c4aec935004631a29ea1382ac3827df603e42a9db000000000000000000000000000000000000000000000000000000000000005b5b + "000000000000001e032540": 25447bdfd63dd3e622671ad29dd6360a085ce6006ac74a6c0d6a3c279ffc3097000000000000000000000000000000000000000000000000000000000000005d5d + 000000000000001f0110: a2000800424573be7b442152cfeb1c3601f99f3130c6f17c7c08b7aaa330c6d87a12515a1f5c82fb7448102c04e67a1975867232ec52a388958eb4e9e6baeee14c5293abe511503fe6ab38d5ac1c8b7bfe8b097205de90a84730a84de1738a85051d4a4a026311da7578e0017433b8084636a698ea93c593baedaf117de2b36e72020115414bbe11 + 000000000000001f0140: 00820900ac90a14f9020bf1306cce40cb69603c44f376b19b3b89b164da4adad2522f3cd1d44b03ea062f4472e00e34d5e0293dd01552dc38a7a87408fd085c93074aa6c411fa71618826a808ad48f199fd524a76e7dd86dbebc0f41833f7831fbc6841282d01caf4063b99148e997961d12ee3dd264572aeb229046a8e5e9b225dc1dfacc72fc0c + 000000000000001f0210: 10517f8d773163c41a3ad0446b196c056622643019f2f0f7b6ee0bd6374e224e000000000000000000000000000000000000000000000000000000000000005e5e + 000000000000001f0247: 473c0e43a2c5fd95e1b6b6154e7a720a5d8d257790f508c601633aff2bd9e48d000000000000000000000000000000000000000000000000000000000000005f5f + 000000000000001f02f3: f38ccb0299ceb194f40be88e74d175c707832dc74a73b2087aa1027689cd40f4000000000000000000000000000000000000000000000000000000000000006060 + "00000000000000200150": 202008a8c9994c2caa7d57098169757caf7127daad05bbc20a0f2482c66e0bd3e0f2425812351e6086b7c92ade4db538680c6d267f479fccb2d4ce9b10062afdd17369911f20e6d61bcc44717172e1e2c6b2ff387448722c2fc17a17726189a0c93514e5dab40e9fa3dba19b7c08af62edc9239b652fb5675abe268ec732eb467087969ba6084b0da64d06efc9a752b9453f768fd58528f039f4fc63aff3e8353f3268ecf15df9771e6fb85f4290b650931131cbbd4f910b772185cd8006f0b917801678a34b4fa7910d + "00000000000000200190": 8005a8a07dd0df391d962fd4150f793cbc18c888b63ed02cf54a3d0db5cb3f2b29c36e240a9fbced1659f7cd1a5c0af9ee189842e1692e65352c1b487f61f2d752f2a0b3d71a59773d2c25f7cb39674f6ae2cbc3fbdd82d1ed6972a6aa3ed016269f3e55502420c817723a3d92295eccbaf6eba511b46779dbe67626cde023cf8cd37f33460654088ca3d9745931ef68100145097c22269521f4df6a11f5b2d3e3488216298a8a7616b99a204e2e9faa22aaf21289c9e4f0495f1a388969a5172bc620b50bfa30ddab146f5a9ea74107c8f4de693c261104144e5bc30d5c84fd24817d46e2335b5833670be16772b9714e64a1009a37324bbe60a52cbe8674e7964b8cf9982e83b33ff6411b + 000000000000002001f0: 881000005ee18ade69f359305864cf1d17d37a06c42736b877fafc52e64c3413655b96b120aaa833785c900c5124bf8ed8d919ee7eaa067d54c92198804d10e1cd19bffbc11f815452859f52a833bb33a0648e223514255bf86b3b57a022a2cc9e761ce5b5d414 + "00000000000000200256": 56caf919aa7245d39566bd66e00d1a91b9894333d5d0384b6a2a12c6e66d95d1000000000000000000000000000000000000000000000000000000000000006161 + "00000000000000200295": 002000086e520d01c9c38a65012be48c7d7a1ee5de3f33497cd4de2605f2998d6f8c0398205c96cbd1a7d9f8be9e808c5e59d525622725bc800f5ca3aeb5e06ef58f96b05c20 + 000000000000002002f1: f1a7fba0a3a8c6f5ea1249d0c556b27da90c998fe9d6dc565911b5638f66f5c4000000000000000000000000000000000000000000000000000000000000006363 + "0000000000000020039560": 956ba33c478bde6c11ea2a1f6105753d55271cad5ce7101a08b8547a234e74c6000000000000000000000000000000000000000000000000000000000000001a1a + 00000000000000200395d0: 95dbef93d0be459e34c6e6663551ebc31befb871daa6430a62329531bde4add7000000000000000000000000000000000000000000000000000000000000006262 + "000000000000002100": 6455555655c0a8dcf08966f67ecba39300a8722ba44482d5e0b23380a9f213e8c337f590cc19227b208a9a19221548e782068f0fdb6fbd4617116bb152b668c3e9a674ea29c41fb49d8c482b50bf07df57e2cc5268959d80fcb3f2da9f0f14d21532f57241f3171e53b19451c176ee59d8e6f3c12b6adbd31add13bdea3cec23766e64033aaf8f4d1b7fa6517eea208326fd635d9ea0a7f08f465d46b631a275df33d13c95e4bf02c51fa96003c38bcdb41045b60ec68b377fcd9be9095d5864500f657183cfe777d1e8206dec6c64316471d87e7d08b832fae1293c5bd921e6b05fa6fccc65e0b959119921c062fd7630074d60ddcf9cdfdabdb10356fa4383f98d61c9fdf92909bcfcb8191e8cedd6f68a9ef7189ace7c6d176c54bd862999e67e1b753e6f91050c255082791957aa7cbdb78645f6e96d2f01c675433421af4d024b10de397ed0c1c85b6cfe032092d380598feb12fe1a1d058ddf0866fe3b7a3ad8fa099914132421d8074030331a5cd30423af28fd201e24c41618d281a3d0cab380970b73da816a8fbe03117c16175ec4b7a0f7cc597669ca4a4d8d7419d400afd56116b898807f03cdc2ef70bf4e1cdf49b5b8b47705368b4d5e009e292ec1c92b10ef50501bbb191072bdd0350f3e1dc4e5b97afa0931090e318bfaebecf8e2a337476e1a8c102dd970969c6688792f1dba8bc88682602128060614528f82d9a2cae1524e615f6a42cf99a99fec431e4520 + "00000000000000210160": 2100880851a6e73460f30029943bf553f273b1a48ccd31437e3f83a25fb38dffb2d6ac280f0724c3ee74e7f23cb76e9395e87d1e4bff63f1da7049a6eca441724797f3874b13cf2fe9fd420120d82d0609329ca9b5ffa91da307e1b7f51057b747c639a34af8174340fb29a51a13adbfe03a805124b70da17033d7707cfbb0dfcd54d24c1e63d421aaadd43d4d5793871733182fe8506eed15062b5875e420c8b7d22cba53ac37850a + 0000000000000021026b: 6b0fd3cfa6040f65f7276c040ccc7a4b246f90de0d99b1d3d43a2feac6c894bd000000000000000000000000000000000000000000000000000000000000006464 +stale_keys: {} + diff --git a/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-8-chunked-commits-pruned.snap b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-8-chunked-commits-pruned.snap new file mode 100644 index 000000000000..e0b2b0f792b3 --- /dev/null +++ b/core/lib/merkle_tree/tests/integration/snapshots/integration__merkle_tree__rocksdb__db-snapshot-8-chunked-commits-pruned.snap @@ -0,0 +1,136 @@ +--- +source: core/lib/merkle_tree/tests/integration/merkle_tree.rs +assertion_line: 658 +expression: "DatabaseSnapshot::new(&raw_db)" +--- +tree: + "00": 0d030c61726368697465637475726506415231364d5405646570746803323536066861736865720a626c616b653273323536 + "00000000000000000200": 0059914596ed2e70745c44ef315747ea29aff72bce6757aca30a434d9bb70781000000000000000000000000000000000000000000000000000000000000000202 + "00000000000000000204": 049dfddb9f03237fed8d902e350f0b0e9b85af93f49fe02f0d189c47cd7b122b000000000000000000000000000000000000000000000000000000000000000707 + 0000000000000001020e: 0edeb726b4588b4b373d38b7b43e3083c6e105c991e1812bdcce6c373ed98dd5000000000000000000000000000000000000000000000000000000000000000a0a + 0000000000000001027c: 7cabd5f57ff72670052939485c4533b01b26fd0e490c31bd48b5ac5002ff1f83000000000000000000000000000000000000000000000000000000000000000909 + 0000000000000001027e: 7ed4dea78574266e019059e5b5fd6f94ed1632bd4a643d1c51aa02974def5684000000000000000000000000000000000000000000000000000000000000000303 + 000000000000000102bb: bbd2fb6ed132cf2780a90802acaaa371de119dc89f636dbb647ccdff8b0dc056000000000000000000000000000000000000000000000000000000000000000d0d + 000000000000000102eb: ebdfe46967031db428c3b807c7f8d78f6a51e9ca5f0500ca6099d3d26b1d312a000000000000000000000000000000000000000000000000000000000000001010 + "00000000000000020207": 07216d95abf0edac40a58f6ca27749af481a4ed3b33dd2d3f10efa814d6da0a7000000000000000000000000000000000000000000000000000000000000001818 + "00000000000000020229": 297faa5f83989e2a8b574589c3eeb5fdced0634fd75d662507604796dd4edd1b000000000000000000000000000000000000000000000000000000000000001515 + 0000000000000002022b: 2b5ffd54bff018009f93f7cd782e8922b24520eb99a100efe2c1a4ae6c1ca3f5000000000000000000000000000000000000000000000000000000000000000b0b + 000000000000000202a5: a58e3a77937b9b747f70d4aee9e84992d7955e52e2de71dc9615df3d16b2b816000000000000000000000000000000000000000000000000000000000000001313 + 000000000000000202ab: ab314b8d202e718d011f9f90df81dd0a00dc4f2279da45121f6dec7257622776000000000000000000000000000000000000000000000000000000000000000f0f + 000000000000000202ea: eaa40f6cdd316711961300a0f242fdc226f42d4740c381f05200092eaf70b841000000000000000000000000000000000000000000000000000000000000001212 + 000000000000000202ef: ef783cc720dbf74c747a155a8975b324d2f8fa80672969dc78fe6f12ea59d03f000000000000000000000000000000000000000000000000000000000000001111 + "00000000000000030293": 9300ef4007f853d076758bff4c00d6c979113664a6c948d0d313daa379607fdd000000000000000000000000000000000000000000000000000000000000002020 + "00000000000000030299": 99fa0e7a995a9b3c03f9a186d9581dd08387846aade5384b8e85c9c7c193dfb3000000000000000000000000000000000000000000000000000000000000001b1b + 000000000000000302b3: b314fe2db0b91091038ce12e45838efdfc5c48fe1e68d4f6a4f990ba9e4324f2000000000000000000000000000000000000000000000000000000000000001f1f + 000000000000000302b6: b63eb46438ece57d3d58c40b8193308ffccd3eedec2d645a5ac181f7783aa9c5000000000000000000000000000000000000000000000000000000000000001c1c + 000000000000000302b9: 002800001632849641ed8fd7f10b2ae3ddaea90de793987e076a54b1d561f0bbc1fa4c65037236b2c6d1e76a12f08ef244830561fd8c3f4cb4eb1d180d9dd3885da41f45cf03 + 000000000000000302dc: dc3e793187abbabede6f6fc80c3ab3ac4a31019d0ef19f59aff10f614a7149fa000000000000000000000000000000000000000000000000000000000000001e1e + 000000000000000303b950: b9592813919e07b8df5fd836d1c6e3ac9a9eb7cb66b5ee8a1f4ee0ff3e0b2508000000000000000000000000000000000000000000000000000000000000001919 + 000000000000000303b960: b96e7e15bcbf96c67b1f26fa5ba80089388fbefad39968132c0791cb313d0157000000000000000000000000000000000000000000000000000000000000000404 + "00000000000000040205": 05bd0bebbb9c224a78e8d9b6876da7f75cc0d247ce0f6e99f8921fbf410a09a4000000000000000000000000000000000000000000000000000000000000002323 + "00000000000000040249": 49ac53849b70d666cc840b4add0baff56fa9ce7e27be2acb275d109a5994ff8d000000000000000000000000000000000000000000000000000000000000000c0c + 0000000000000004025d: 5daa965d9688be0ac629916a9920e45910c13d1fe45d257a9e17217f226dfb4d000000000000000000000000000000000000000000000000000000000000000101 + 0000000000000004025f: 5f8a83305cccf521595d4bdb1ec8b03be3e9e8e27438db250b9e89f09981799d000000000000000000000000000000000000000000000000000000000000002828 + 0000000000000004026d: 6dfea072e8e999ba0adb48fc1284af16cc1351d53e52b175f9dfe153602b4362000000000000000000000000000000000000000000000000000000000000001414 + 0000000000000004029e: 9e262dd28666dbdb4101e1c27a64deda0e6064360be4c53c89e7b2b35a311943000000000000000000000000000000000000000000000000000000000000002222 + 000000000000000402b8: b860ff2535a62373c8c7aa6a31b436cb16427464bde987fc0c35238e10663a56000000000000000000000000000000000000000000000000000000000000002424 + 000000000000000402dd: 00280000bf3a7497054eac25ada2fc8c4d315a3a240e6bbe6ecdd3f2fc0f82d8750984410457d9159f07e3d3466d61ab4c26d5b3ea993669fa9b133ee2c9c0a4b859af98b304 + "000000000000000402e8": 0800800005cd913895621437d7b5a41c4cc473ebe257221afa3d6e70297ceacdedfefb2a0437abb18273775dc4a8d30143a3bf7e2db1495570bec8a7876a34f4c18eb9d8eb04 + 000000000000000403dd50: dd55470824e0db2b94ea10ae29afd473f265bbe758854354b926846821fef91a000000000000000000000000000000000000000000000000000000000000000606 + 000000000000000403dd60: dd69263d904a01d711be57a6f4177bbf6745084517b2ac16468ab075cb88a962000000000000000000000000000000000000000000000000000000000000002727 + "000000000000000403e810": e81cc9e81f01ea7314bf83dc9b7fd942974427fe130fe80b3fb11eed3f80d4ed000000000000000000000000000000000000000000000000000000000000002525 + 000000000000000403e8b0: e8b8b981f358516ba6e7b76e0007bdecf3e97873abe468fbef40110d8206c8d8000000000000000000000000000000000000000000000000000000000000000505 + "00000000000000050201": 01ad1cfd47cdacf2f23714b10102071ea05f91e25ffcf10ac043964bb004b83f000000000000000000000000000000000000000000000000000000000000002c2c + 0000000000000005022d: 2dd4a3d0d6d98198be8e16968d57cda367833b99cfd88c809ce3960ff8b41aba000000000000000000000000000000000000000000000000000000000000002929 + "00000000000000050238": 38ec53adc1cd8bc9f788a5986f73d4e29e2d98945c0aa1d6727be9f8baba4337000000000000000000000000000000000000000000000000000000000000000e0e + 0000000000000005023d: 3d8a8b22175714443f990b996dd26dc71cb53e030d1bf48844df12486e5d4f2a000000000000000000000000000000000000000000000000000000000000002a2a + "00000000000000050259": 5970e9fe810ebec7d53c18e51fe72b464e75851550b0c34a5e7ebabed8e6c7cc000000000000000000000000000000000000000000000000000000000000002d2d + "00000000000000050260": 00008800b450bcb1628ab1ce5b83630fce3da51e983f02c587d8eba576f264288f799bd2055bf094efc1e3fb61fa5a849f862e091e65738a95d239372e92191d2f5779752b05 + 000000000000000502c0: c0519bcddf9b31ea9fbdfe61998e503dca06026d059b990948d1e076657a191e000000000000000000000000000000000000000000000000000000000000002f2f + 000000000000000502c4: c4fbca1f402303e4bc0c85ed3817bc2ff549f665047388b1b40fab23553b2a9d000000000000000000000000000000000000000000000000000000000000001717 + 000000000000000502d9: d9898cd6484f797318b72190d1939771527071cac4c2e9da4e053e2b6e0dfe1a000000000000000000000000000000000000000000000000000000000000002b2b + "0000000000000005036090": 609f2dcc908d758e6b7dfb10333930babc58eb6ea82a7fb7e332371ba173d855000000000000000000000000000000000000000000000000000000000000002121 + 00000000000000050360b0: 60b22e6aff3665a5256eb5974025b20ba28c2b93ef016a66812f4c6e0c8ef5c4000000000000000000000000000000000000000000000000000000000000002e2e + "00000000000000060203": 03484fcec5bd9ccd1b448f7839fe71c8fb7d0e9e9f9076447daa552c89691f60000000000000000000000000000000000000000000000000000000000000003636 + "00000000000000060212": 121a1293f56198bb1639951b344a3ae26f5bca796569bb07d53ce98f4085bcbc000000000000000000000000000000000000000000000000000000000000000808 + "00000000000000060213": 1303216ea6a1a1b396f5fe43017e9869938a896901ed852d95079a1a7d7d57b5000000000000000000000000000000000000000000000000000000000000003434 + "00000000000000060219": 193fbc2a619e98d253554723ff80e1c2de926a8547f09ee1568e53db2b0fa0a2000000000000000000000000000000000000000000000000000000000000003535 + 000000000000000602a0: a0aab27f895e1a504ee0862d8a90c2df63bcc4311409bf357a09d6ba4311d0f8000000000000000000000000000000000000000000000000000000000000003333 + 000000000000000602bd: bd11589aa8ca58a6c00ed1b1354d30b53279703644bed0a73e63513ad11fa5bf000000000000000000000000000000000000000000000000000000000000003737 + 000000000000000602c7: c78049d38618be10de663e1ef60a5a34a9e635692bf9952c3afb837af3f66e1d000000000000000000000000000000000000000000000000000000000000003232 + "0000000000000006032560": 25659d69d133a28e77d07372508d9bebf0784d709c5f3bfaf1a4d53e22d4febb000000000000000000000000000000000000000000000000000000000000003838 + "0000000000000006032580": 2585f6a0139e0a71940d088e01221a4a5a6852220a894f135cf5c415c511f247000000000000000000000000000000000000000000000000000000000000003030 + 0000000000000007020b: 0bebf05465e75ecbdaadb3e79813a435c75b125b2679361a673af1f16b0b8ac8000000000000000000000000000000000000000000000000000000000000004040 + 0000000000000007022e: 2e54be9fbeefab49441556d63666897e33eedb3ff99ed9bffe4b88f9fd85ba11000000000000000000000000000000000000000000000000000000000000003f3f + "00000000000000070252": 5245608ce703d4a307b8c83f268dc60ac248dc26a891baa6116feff1e3cd8ba4000000000000000000000000000000000000000000000000000000000000003939 + "00000000000000070262": 620975a9aee0d240d9f52ead3ff4f04b9043e260d67bb961fc16188cf1f63635000000000000000000000000000000000000000000000000000000000000003a3a + 0000000000000007029b: 9b420dfe14cdceb77e4464b2a841b6cf6221a29acf6bf363a556a13b72467404000000000000000000000000000000000000000000000000000000000000003d3d + "000000000000000702e2": e2adc6f93c6726bb6f0f56324add68457a28b691aa6e08951cab65ec6f37b54b000000000000000000000000000000000000000000000000000000000000003c3c + 000000000000000702f6: 0082000048902a08b0d7edf4969a608d61ec9911074fc5efb163bc207048d35be9ac0503071993dcbaa58bdd07fccd22a725289a2ab266ee265a3c0dc0eda3a48a154235a107 + 000000000000000703d8d0: d8d8c592cc9c74820b9ed9a20fb5709a0d9273f210fbfab39feb94c62f00d806000000000000000000000000000000000000000000000000000000000000003b3b + 000000000000000703d8e0: d8ecaa095dc3b2f77c057c770718c0affe6a9caa0104136143437cf30fb5688e000000000000000000000000000000000000000000000000000000000000003131 + 000000000000000703f640: f6479d69073315ce4df8b1234bd36fde45581f7b61749ba87374c8c71f0b0d72000000000000000000000000000000000000000000000000000000000000001616 + 000000000000000703f670: f673a7125608b9951ba643a43a2b7fc3a87af522e820cccfa65f977949a98c12000000000000000000000000000000000000000000000000000000000000003e3e + 000000000000000801b0: 8224860824620206f5ae3438e207a4bac199dfceedf205d17a5a7f5396b7ceef9eafd0a0082c0a644c823c91a5bea9558492e3b48fd18be253175f64636dbcc860eeee5865037213670135f5ec0a5dfe384bc455d0d377e1c023b39bb0dd1e27e2caaa49a81808c7d6f9eacac49c7dd8949482043c0c42aa0cc5450e38c51e623ed0327ebf094103f3a8e759576757a9c487914a62da083898a037e4aeaa1944c08fe9ff0cfe6a1604409271ef93d4db4c3128801b7e9ae6d181099ca7b8a849670cdc5106b8dad4b803332f352a2bf593288dda47c506db7903535e1900681d84c1bd7cc0ff2fd1548b0134638675b595022e38b43d20ae971594edf5e202eeea2a58bd30f692b2450e1506 + "00000000000000080269": 69d819309016b97574c9f345296c54f1432e1a4fc9e48fcc3d3bc03bf4997fda000000000000000000000000000000000000000000000000000000000000004747 + 0000000000000008029a: 9a559085a559a2e90b54fb02fdd4fd74ab5d604e41c4888c170cdc87c296408e000000000000000000000000000000000000000000000000000000000000004343 + 000000000000000802b0: b0e5ace49939a244a59f3654a48da2d9b1c9eb3977925c61a3bac41bfdcfb4ab000000000000000000000000000000000000000000000000000000000000004141 + 000000000000000802b5: 00080200e421f147ed76b69035b1dc7a21a4896daf523abd3a05594f0b9ebe63488ee5700877ab4544f0f4d2663fcc4619ce5a1dd6020cb5f2a4751df29fbb57892276342208 + 000000000000000802c3: c3a199feeed885aebcd589011ee28fdab9550fa9fe53320e56803a6cd4aa0aab000000000000000000000000000000000000000000000000000000000000004242 + 000000000000000802d1: d1e5b3c0fc106542ae1485cb8b21e43c3c13630a6a3c20f8615c4b7cdb572490000000000000000000000000000000000000000000000000000000000000004444 + 000000000000000802d8: 000000a821be1d978695211e8e63aa6efccbdf85110a0879cc45f718ddbef1d8392221f80705d8b58aa36651d72ad489469fee23b99bdeab996c96f9aaa1fb691e52b0bdbe079e3d820be42fcb2ad42a93a18226f5eef9ae386b6e926111d97489670383cea108 + 000000000000000803b550: b553466aa2101212ae2cd93d024063d8367439fdba7b959b730db13a4377f992000000000000000000000000000000000000000000000000000000000000004545 + 000000000000000803b580: b58afc68cab79b077af15ab3b543a110771b218b2c3dd3dc097812e86a89629c000000000000000000000000000000000000000000000000000000000000004848 + 000000000000000803d8f0: d8f4c0f3150c5cc8885c2c0aecbc6819e6689bf4338c70da4816d4cb66ed94f5000000000000000000000000000000000000000000000000000000000000004646 + "00000000000000090100": 8a8a882257b5edb42fb6c1d47b66df6f451d1f0b07c707b265b47a0a445a49121bf17f09003564f8fd0ce402f3344ebabc39f73df9f5c2c5e7a6ef60d3c65bd3fa126e386b05663d14857c3a5444a050d9655e3827c4728e80d79d1292fd51f52c34b1e7457b0614c2aaac166403626fff92472cec22712462ccd14ca9d431b2cdcc75fe0912cd0026e57d3219488c374ae9bd6c9778ffd012f013335e68a58e614ab7b397b6b3db04941121db179d2296e2f1633f191f7f52dbb69956adb74e105be899b9ab90771c02eb7ced5baec3075725903e2e21b0e1457be9ff668e5e6d40e843117004e4acfa09ae37c191db8c160b309ddf0f969c234bb77fcdf25dfde6f541e0ddffbcfe0de2076f21fa6ab54c46ae0467b9a707bc397efcc6b392f748ee0417583a5a1eb326d6092c9ae2b72aadaed919d02086f2daedba2939863499c8c7a63e1eef851f57dc9901 + "00000000000000090180": 89069ac4c9e498cf8311cdc94afb4afac57857c4d11ff4cfff721be3f2f627b2000000000000000000000000000000000000000000000000000000000000004d4d + "00000000000000090209": 0959c7836901cba04c833b419ea0dfe0c205bb32a67214c6445b61d7b4a0639b000000000000000000000000000000000000000000000000000000000000004a4a + 0000000000000009020c: 0cbf607275c4dca4df0f7ffbb643193ab7a9d7419c0f02e0c3ba79f398f81d46000000000000000000000000000000000000000000000000000000000000004c4c + "00000000000000090239": 39df927e1d5265409744353547dab1f8a27444b2fb917ee896983e046ba91163000000000000000000000000000000000000000000000000000000000000004f4f + "00000000000000090277": 7783908d59c75dfbcf114da6847cb8abd3be94593bb4267a9b1ebfb223b98ea5000000000000000000000000000000000000000000000000000000000000004b4b + "00000000000000090294": 80000008170ede3e1920837d4865c7c6087bb6117d6083871d773aef1916f8bf1662faea09127afe9fa81b7edde8ad3c48f164abaf9de3ea3c04b5e7eec32bc2609a803c6909 + 000000000000000902a8: a8c56ac32ee75c657919ae3d71f58d15ac0eb2e9d2ff7b9c04b8fd0f96ed7448000000000000000000000000000000000000000000000000000000000000004949 + 000000000000000902c5: c54f0faa1e221486827abe3f900feb05956f20fc7f15f4e084bd673c489aada4000000000000000000000000000000000000000000000000000000000000004e4e + "0000000000000009039430": 943a91e688714abcd78bec435e5d1f6412e5132eb61ab16fb71947c26d6dff64000000000000000000000000000000000000000000000000000000000000005050 + 00000000000000090394d0: 94d67b778ebe500b3516f7d892d44cb281067e662cc1483d559cd0d73e00f177000000000000000000000000000000000000000000000000000000000000001d1d + 000000000000000a0130: 00000a28803101a64d85ff96891a1929b5410e7a7a0b6a60ff8bf3378008164e6c21ae82051d8a35b505566077f9e13da0d84b530d3092ab261a2d2abee46f7f28d74c8e20097d37b6b37acd4cf4045a574c36ae84c9a1dd5f69f307c91f0e72fb20bf0d4fa8053c4e03f48d247afced2623d420be80ad04a1a9e87e27d40dde7cea4f96a9a7180a + 000000000000000a01a0: 020882086b44da2a9dd1eaf1e18dad37d7e76005d4b035d5c928629d180c9e0d15554bfb06182fc339100ba14fc2bcbb0ae880c957f3ab0d069eb5f206dd4649004c3fb63102adf6e932d9d77affa57a5662c9976af696c390b173f13f9accdb45d2b3245898090d19ead21c451a82ac4ff7cd7b2f004013811ced4ca96177379ebf5ccc58352902ce290b979ca575bdc45250d691ce3ed7014ec2e83477608208bb75cdead773490a + 000000000000000a01c0: 828a2000c44c8b773c992751a9a68cf562a5d66a82ecb3e5b1f3e825583af4a4dc7188fb05f8e73c81b1ec28c666c9a504eb2be2aa8a2cc1cf0af2889447eea287118568bf082de02cba921be88fba820ac33a520ad265be4c4a290d864f0377884abf900ac005e8927e187971ca77095814caa6cc732835445a638edb782610240ae86388c788090ba1131ae7242bf1d1dfc8c0f32eb0a5cf80c6f85dab3d0d0c5355881ead0e86062e3b99b6d3c2022157528163ef99f3fb81204f8d3f32ff3bcd8fb918e40919090a + 000000000000000a023e: 3e5f86ed85e39b54418692f15640a6bcaf76b7cb6f7b6548da28177df9232ec8000000000000000000000000000000000000000000000000000000000000005252 + 000000000000000a0244: 446c0030d10b7a6b6c2970f02f4505d71de7bbf95c0c99884c0c01467f53a8ab000000000000000000000000000000000000000000000000000000000000005858 + 000000000000000a0248: 0008002091380386034a68521194d26b451f80c900eed4fb13cb9e8913e3dfc5e592b00a0a52d96f4eeda24e81dd90939b2748dacd8a765a938bde5574cc54544aa00d194d0a + 000000000000000a0278: 78421e310f5e842ec956e626534d27008b85f06736ae1f67a1aaa7ed7f21c68f000000000000000000000000000000000000000000000000000000000000005353 + 000000000000000a029f: 9f654242a774697de6c44df46fa9670371ac5999c0b14874dafefb8f4d60638a000000000000000000000000000000000000000000000000000000000000005454 + 000000000000000a02ad: ad72cdcdcfb4f0632d055d5445137abf73f21c0dfac21add822c71e82bf2353e000000000000000000000000000000000000000000000000000000000000005151 + 000000000000000a02ca: ca19a24e1132928718d987a578a4ca038eab9602e8c4abf553bd7189459effef000000000000000000000000000000000000000000000000000000000000005656 + 000000000000000a02e1: e13b9b67638cdad23dec7259168f9562d1dc889b2be70d752bc3dfe82bc6a712000000000000000000000000000000000000000000000000000000000000005757 + 000000000000000a034850: 4858ed2508d9a8a36c8c771fd9cdd71285d5c4dc624a1e63c0abd64cef699b03000000000000000000000000000000000000000000000000000000000000002626 + 000000000000000a0348e0: 48e2e8edc61fc5bcddff86b114b5a870369db85287fe39ee8b9a63837552c315000000000000000000000000000000000000000000000000000000000000005555 + 000000000000000b0110: a2000800424573be7b442152cfeb1c3601f99f3130c6f17c7c08b7aaa330c6d87a12515a0b5c82fb7448102c04e67a1975867232ec52a388958eb4e9e6baeee14c5293abe506503fe6ab38d5ac1c8b7bfe8b097205de90a84730a84de1738a85051d4a4a026306da7578e0017433b8084636a698ea93c593baedaf117de2b36e72020115414bbe06 + 000000000000000b0120: 00048828f9c73b9300e34ac8e315a7bff566e806e513bcf859f73fed008bbc294cbd86170bfa858129a21ac49dc254a945b31dfe16d6aa38f787508a37e17770d646efc8440254811a6dbdccae4f94ec1115b47b52a93363dbfaa0997714af738a41b673af93028f46a18e5863c4ada0e7cd39134e090bc980515a25e481ffc3763ff91c26e91405fdcebb4312eac9d2281efff9ac70c4693fe3a7e5d1be16a233845059ce1d9e3e07 + 000000000000000b0140: 00820900ac90a14f9020bf1306cce40cb69603c44f376b19b3b89b164da4adad2522f3cd0a44b03ea062f4472e00e34d5e0293dd01552dc38a7a87408fd085c93074aa6c410ba71618826a808ad48f199fd524a76e7dd86dbebc0f41833f7831fbc6841282d00aaf4063b99148e997961d12ee3dd264572aeb229046a8e5e9b225dc1dfacc72fc04 + 000000000000000b0170: 0080822205a922b3f8195dca99196eacd675b9eb833f50040232bb1f618834baf5b38ba109167a1e11488fdfa9d5b12121dd9910adfbf852c209f3a577b967000745f350f60aaacca0ce4f75eee32589c24eec3958eeccef822af3b15daaa432fe3e82a05cb80bdab1eee1892b8092343e6487bc31fa80fb2dd7cbb9ed1c6fae878f83dded1c2d01060b08b1a2e058cb5ec3f70f413c5d5440ef7d6e542223d3784fbd6c72e8c68c01 + 000000000000000b01d0: 08800906f4063981ea97ecdb06015df9bf4f6727c57efbf4b0646c3476deb36794324db108afd85ebff302f5bbba6058397c4ea0828d22a3778df3f112311c1d0accf7c7080b2b1948ebf5edfef56db3a187d77bc30e4b3200612376c93a909b907ff574c6e5087eae10ad154a3d3f012dcfce2fe0adf14890f879676eb80b621029c68d316ca305d82817436305281ec13d95108dab7f12d2efe6baeef0e20c192522e3fb9acfd20390931c85595b1e71a6693a36c92ac5c7859148c77e098b7c950389ab74c5cfcb04 + 000000000000000b01e0: 2880a18043fe9188b8b985568354bcdd9250b431c4e714f6b42051be284857a8d2be17220a5a0aecb3ba406c7d3e437d8ef8e625e2d77e062da840461ccf48c0c4308f403407299f131a3be0248d2e798e4ad2f600e15e0f33a6cd627aded2521f4ec15f99780bf11b8ad362ee769da41a1cd9c52ad888b677eec1f20671001f8777faa9cf964a04fc7d7cf7cf7a7beefa4a24a16c1aee39a859559a0af0057b8ee2d08190b6f92002ac0b59ebc0d792db7fc244e10d924fca664b6f6e2ceec70264212ddc795c962f0112c7eac2be7db3066f6ac802ba777e9eb44c7a5cab845977367ad8b34df71bc502 + 000000000000000b0210: 10517f8d773163c41a3ad0446b196c056622643019f2f0f7b6ee0bd6374e224e000000000000000000000000000000000000000000000000000000000000005e5e + 000000000000000b0225: 0022020063a7ef2de46a7ccd3a1fa06a407e928949b80e72ddb6b3db95ec8715acd6f7f70b75611c7ae282df1b3ad74bb8b08fc01ee7f4dd46036cf91eb514376122e4a62306592ab1d5500c2ed989fa665f501c67499af68a98a0090ea4c7e66cf53684fc0c06 + 000000000000000b0247: 473c0e43a2c5fd95e1b6b6154e7a720a5d8d257790f508c601633aff2bd9e48d000000000000000000000000000000000000000000000000000000000000005f5f + 000000000000000b025e: 5e3c1c034ab4b4eead015b30d1a272db741bb510d9e3d3f1abd099652302414f000000000000000000000000000000000000000000000000000000000000005c5c + 000000000000000b027b: 7b26af81f4c1ab7cdaa6fc0c4aec935004631a29ea1382ac3827df603e42a9db000000000000000000000000000000000000000000000000000000000000005b5b + 000000000000000b02d7: d753f729b5ccfe040f4f592b95b710c4e6f147bb9178880080f2d094becfad86000000000000000000000000000000000000000000000000000000000000005a5a + 000000000000000b02e7: e7df81ef1c550b915e5e9060719aa92ac78a1fddd47c325b79435a7752f7e5b7000000000000000000000000000000000000000000000000000000000000005959 + 000000000000000b02f3: f38ccb0299ceb194f40be88e74d175c707832dc74a73b2087aa1027689cd40f4000000000000000000000000000000000000000000000000000000000000006060 + 000000000000000b032540: 25447bdfd63dd3e622671ad29dd6360a085ce6006ac74a6c0d6a3c279ffc3097000000000000000000000000000000000000000000000000000000000000005d5d + 000000000000000c00: 6455555655c0a8dcf08966f67ecba39300a8722ba44482d5e0b23380a9f213e8c337f590cc09227b208a9a19221548e782068f0fdb6fbd4617116bb152b668c3e9a674ea29c40bb49d8c482b50bf07df57e2cc5268959d80fcb3f2da9f0f14d21532f57241f3170b53b19451c176ee59d8e6f3c12b6adbd31add13bdea3cec23766e64033aaf8f4d0a7fa6517eea208326fd635d9ea0a7f08f465d46b631a275df33d13c95e4bf02c50ba96003c38bcdb41045b60ec68b377fcd9be9095d5864500f657183cfe777d1e80c6dec6c64316471d87e7d08b832fae1293c5bd921e6b05fa6fccc65e0b95911990cc062fd7630074d60ddcf9cdfdabdb10356fa4383f98d61c9fdf92909bcfcb8190b8cedd6f68a9ef7189ace7c6d176c54bd862999e67e1b753e6f91050c255082790957aa7cbdb78645f6e96d2f01c675433421af4d024b10de397ed0c1c85b6cfe030c92d380598feb12fe1a1d058ddf0866fe3b7a3ad8fa099914132421d8074030330a5cd30423af28fd201e24c41618d281a3d0cab380970b73da816a8fbe03117c16085ec4b7a0f7cc597669ca4a4d8d7419d400afd56116b898807f03cdc2ef70bf4e0adf49b5b8b47705368b4d5e009e292ec1c92b10ef50501bbb191072bdd0350f3e0bc4e5b97afa0931090e318bfaebecf8e2a337476e1a8c102dd970969c6688792f0bba8bc88682602128060614528f82d9a2cae1524e615f6a42cf99a99fec431e450c + 000000000000000c0150: 202008a8c9994c2caa7d57098169757caf7127daad05bbc20a0f2482c66e0bd3e0f2425807351e6086b7c92ade4db538680c6d267f479fccb2d4ce9b10062afdd17369911f0ce6d61bcc44717172e1e2c6b2ff387448722c2fc17a17726189a0c93514e5dab4059fa3dba19b7c08af62edc9239b652fb5675abe268ec732eb467087969ba6084b04a64d06efc9a752b9453f768fd58528f039f4fc63aff3e8353f3268ecf15df9770b6fb85f4290b650931131cbbd4f910b772185cd8006f0b917801678a34b4fa79104 + 000000000000000c0160: 2100880851a6e73460f30029943bf553f273b1a48ccd31437e3f83a25fb38dffb2d6ac28050724c3ee74e7f23cb76e9395e87d1e4bff63f1da7049a6eca441724797f3874b07cf2fe9fd420120d82d0609329ca9b5ffa91da307e1b7f51057b747c639a34af8084340fb29a51a13adbfe03a805124b70da17033d7707cfbb0dfcd54d24c1e63d40caaadd43d4d5793871733182fe8506eed15062b5875e420c8b7d22cba53ac378504 + 000000000000000c0190: 8005a8a07dd0df391d962fd4150f793cbc18c888b63ed02cf54a3d0db5cb3f2b29c36e24039fbced1659f7cd1a5c0af9ee189842e1692e65352c1b487f61f2d752f2a0b3d70959773d2c25f7cb39674f6ae2cbc3fbdd82d1ed6972a6aa3ed016269f3e5550240cc817723a3d92295eccbaf6eba511b46779dbe67626cde023cf8cd37f33460654038ca3d9745931ef68100145097c22269521f4df6a11f5b2d3e3488216298a8a7608b99a204e2e9faa22aaf21289c9e4f0495f1a388969a5172bc620b50bfa30ddab076f5a9ea74107c8f4de693c261104144e5bc30d5c84fd24817d46e2335b58336704e16772b9714e64a1009a37324bbe60a52cbe8674e7964b8cf9982e83b33ff6410a + 000000000000000c01f0: 881000005ee18ade69f359305864cf1d17d37a06c42736b877fafc52e64c3413655b96b10caaa833785c900c5124bf8ed8d919ee7eaa067d54c92198804d10e1cd19bffbc10b815452859f52a833bb33a0648e223514255bf86b3b57a022a2cc9e761ce5b5d407 + 000000000000000c0256: 56caf919aa7245d39566bd66e00d1a91b9894333d5d0384b6a2a12c6e66d95d1000000000000000000000000000000000000000000000000000000000000006161 + 000000000000000c026b: 6b0fd3cfa6040f65f7276c040ccc7a4b246f90de0d99b1d3d43a2feac6c894bd000000000000000000000000000000000000000000000000000000000000006464 + 000000000000000c0295: 002000086e520d01c9c38a65012be48c7d7a1ee5de3f33497cd4de2605f2998d6f8c03980c5c96cbd1a7d9f8be9e808c5e59d525622725bc800f5ca3aeb5e06ef58f96b05c0c + 000000000000000c02f1: f1a7fba0a3a8c6f5ea1249d0c556b27da90c998fe9d6dc565911b5638f66f5c4000000000000000000000000000000000000000000000000000000000000006363 + 000000000000000c039560: 956ba33c478bde6c11ea2a1f6105753d55271cad5ce7101a08b8547a234e74c6000000000000000000000000000000000000000000000000000000000000001a1a + 000000000000000c0395d0: 95dbef93d0be459e34c6e6663551ebc31befb871daa6430a62329531bde4add7000000000000000000000000000000000000000000000000000000000000006262 +stale_keys: {} + diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 5804890ac663..c6e3e8df285c 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -531,6 +531,14 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "blake2" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e#1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "blake2-rfc_bellman_edition" version = "0.0.1" @@ -615,7 +623,7 @@ source = "git+https://github.com/matter-labs/era-boojum?branch=main#2771569baab9 dependencies = [ "arrayvec 0.7.4", "bincode", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "const_format", "convert_case 0.6.0", "crossbeam 0.8.2", @@ -633,8 +641,8 @@ dependencies = [ "rand 0.8.5", "rayon", "serde", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", "tracing", "unroll", @@ -1649,7 +1657,7 @@ dependencies = [ "regex", "serde", "serde_json", - "sha3 0.10.6", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", "uint", ] @@ -2149,7 +2157,7 @@ dependencies = [ "rsa", "serde", "serde_json", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "thiserror", "time", "tokio", @@ -2713,7 +2721,7 @@ dependencies = [ "cfg-if 1.0.0", "ecdsa", "elliptic-curve", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -3017,6 +3025,26 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "multivm" +version = "0.1.0" +dependencies = [ + "anyhow", + "hex", + "itertools 0.10.5", + "once_cell", + "thiserror", + "tracing", + "vise", + "zk_evm 1.3.1", + "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.3-rc1)", + "zksync_contracts", + "zksync_state", + "zksync_system_constants", + "zksync_types", + "zksync_utils", +] + [[package]] name = "native-tls" version = "0.2.11" @@ -3661,7 +3689,7 @@ checksum = "1df74e9e7ec4053ceb980e7c0c8bd3594e977fde1af91daba9c928e8e8c6708d" dependencies = [ "once_cell", "pest", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -4369,7 +4397,7 @@ source = "git+https://github.com/matter-labs/rescue-poseidon.git?branch=poseidon dependencies = [ "addchain", "arrayvec 0.7.4", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "boojum", "byteorder", "derivative", @@ -4392,7 +4420,7 @@ source = "git+https://github.com/matter-labs/rescue-poseidon#d059b5042df5ed80e15 dependencies = [ "addchain", "arrayvec 0.7.4", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder", "franklin-crypto 0.0.5 (git+https://github.com/matter-labs/franklin-crypto?branch=dev)", "num-bigint 0.3.3", @@ -4956,6 +4984,16 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=1731ced4a116d61ba9dc6ee6d0f38fb8102e357a#1731ced4a116d61ba9dc6ee6d0f38fb8102e357a" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.7", +] + [[package]] name = "sha3" version = "0.9.1" @@ -4978,6 +5016,15 @@ dependencies = [ "keccak", ] +[[package]] +name = "sha3" +version = "0.10.6" +source = "git+https://github.com/RustCrypto/hashes.git?rev=7a187e934c1f6c68e4b4e5cf37541b7a0d64d303#7a187e934c1f6c68e4b4e5cf37541b7a0d64d303" +dependencies = [ + "digest 0.10.7", + "keccak", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -4993,7 +5040,7 @@ version = "0.1.0" source = "git+https://github.com/matter-labs/era-shivini.git?branch=main#2c73a10ff0310e7f99a0d9a8cea043e697eaa5e4" dependencies = [ "bincode", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "boojum", "boojum-cuda", "circuit_definitions", @@ -5002,7 +5049,7 @@ dependencies = [ "hex", "rand 0.8.5", "serde", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", ] @@ -5209,7 +5256,7 @@ dependencies = [ "serde", "serde_json", "sha-1", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", "sqlformat", "sqlx-rt", @@ -5235,7 +5282,7 @@ dependencies = [ "quote 1.0.33", "serde", "serde_json", - "sha2 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "sqlx-core", "sqlx-rt", "syn 1.0.109", @@ -5393,11 +5440,11 @@ dependencies = [ "rand 0.4.6", "rescue_poseidon 0.4.1 (git+https://github.com/matter-labs/rescue-poseidon)", "serde", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", - "zk_evm 1.3.3", - "zkevm_opcode_defs", + "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?branch=v1.3.3)", + "zkevm_opcode_defs 1.3.2", ] [[package]] @@ -6022,25 +6069,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "vm" -version = "0.1.0" -dependencies = [ - "anyhow", - "hex", - "itertools 0.10.5", - "once_cell", - "thiserror", - "tracing", - "vise", - "zk_evm 1.3.3", - "zksync_config", - "zksync_contracts", - "zksync_state", - "zksync_types", - "zksync_utils", -] - [[package]] name = "wait-timeout" version = "0.2.0" @@ -6375,6 +6403,38 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +[[package]] +name = "zk_evm" +version = "1.3.1" +source = "git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.1-rc2#0a7c775932db4839ff6b7fb0db9bdb3583ab54c0" +dependencies = [ + "blake2 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=1f727ce37ff40fa0cce84eb8543a45bdd3ca4a4e)", + "k256", + "lazy_static", + "num 0.4.1", + "serde", + "serde_json", + "sha2 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=1731ced4a116d61ba9dc6ee6d0f38fb8102e357a)", + "sha3 0.10.6 (git+https://github.com/RustCrypto/hashes.git?rev=7a187e934c1f6c68e4b4e5cf37541b7a0d64d303)", + "static_assertions", + "zkevm_opcode_defs 1.3.1", +] + +[[package]] +name = "zk_evm" +version = "1.3.3" +source = "git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.3-rc1#fe8215a7047d24430ad470cf15a19bedb4d6ba0b" +dependencies = [ + "anyhow", + "lazy_static", + "num 0.4.1", + "serde", + "serde_json", + "static_assertions", + "zk_evm_abstractions", + "zkevm_opcode_defs 1.3.2", +] + [[package]] name = "zk_evm" version = "1.3.3" @@ -6387,7 +6447,7 @@ dependencies = [ "serde_json", "static_assertions", "zk_evm_abstractions", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2", ] [[package]] @@ -6402,7 +6462,7 @@ dependencies = [ "serde_json", "static_assertions", "zk_evm_abstractions", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2", ] [[package]] @@ -6413,7 +6473,7 @@ dependencies = [ "anyhow", "serde", "static_assertions", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2", ] [[package]] @@ -6428,11 +6488,11 @@ dependencies = [ "nom", "num-bigint 0.4.4", "num-traits", - "sha3 0.10.6", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "smallvec", "structopt", "thiserror", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2", ] [[package]] @@ -6452,7 +6512,18 @@ dependencies = [ "serde", "serde_json", "smallvec", - "zkevm_opcode_defs", + "zkevm_opcode_defs 1.3.2", +] + +[[package]] +name = "zkevm_opcode_defs" +version = "1.3.1" +source = "git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.1#00d4ad2292bd55374a0fa10fe11686d7a109d8a0" +dependencies = [ + "bitflags 1.3.2", + "ethereum-types 0.14.1", + "lazy_static", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -6461,12 +6532,12 @@ version = "1.3.2" source = "git+https://github.com/matter-labs/era-zkevm_opcode_defs.git?branch=v1.3.2#c7ab62f4c60b27dfc690c3ab3efb5fff1ded1a25" dependencies = [ "bitflags 2.4.0", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "ethereum-types 0.14.1", "k256", "lazy_static", - "sha2 0.10.6", - "sha3 0.10.6", + "sha2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", + "sha3 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -6492,7 +6563,7 @@ dependencies = [ "sync_vm", "test-log", "tracing", - "zk_evm 1.3.3", + "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?branch=v1.3.3)", "zkevm-assembly", ] @@ -6615,7 +6686,7 @@ name = "zksync_crypto" version = "0.1.0" dependencies = [ "base64 0.13.1", - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "hex", "once_cell", "serde", @@ -6643,9 +6714,9 @@ dependencies = [ "tokio", "tracing", "vise", - "zksync_config", "zksync_contracts", "zksync_health_check", + "zksync_system_constants", "zksync_types", "zksync_utils", ] @@ -6923,11 +6994,28 @@ dependencies = [ "vise", ] +[[package]] +name = "zksync_system_constants" +version = "0.1.0" +dependencies = [ + "anyhow", + "bigdecimal", + "hex", + "num 0.3.1", + "once_cell", + "serde", + "serde_json", + "url", + "zksync_basic_types", + "zksync_contracts", + "zksync_utils", +] + [[package]] name = "zksync_types" version = "0.1.0" dependencies = [ - "blake2 0.10.6", + "blake2 0.10.6 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", "codegen 0.1.0", "ethereum-types 0.12.1", @@ -6942,12 +7030,12 @@ dependencies = [ "serde_with", "strum", "thiserror", - "zk_evm 1.3.3", + "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.3-rc1)", "zkevm_test_harness 1.3.3", "zksync_basic_types", - "zksync_config", "zksync_contracts", "zksync_mini_merkle_tree", + "zksync_system_constants", "zksync_utils", ] @@ -6968,7 +7056,7 @@ dependencies = [ "tokio", "tracing", "vlog", - "zk_evm 1.3.3", + "zk_evm 1.3.3 (git+https://github.com/matter-labs/era-zk_evm.git?tag=v1.3.3-rc1)", "zksync_basic_types", ] @@ -7004,6 +7092,7 @@ dependencies = [ "futures 0.3.28", "hex", "metrics", + "multivm", "prometheus_exporter", "rand 0.8.5", "serde", @@ -7013,7 +7102,6 @@ dependencies = [ "tracing", "vk_setup_data_generator_server_fri", "vlog", - "vm", "zk_evm 1.4.0", "zkevm_test_harness 1.4.0", "zksync_config", @@ -7024,6 +7112,7 @@ dependencies = [ "zksync_prover_utils", "zksync_queued_job_processor", "zksync_state", + "zksync_system_constants", "zksync_types", "zksync_utils", ]