feat: Implement dynamic L2-to-L1 log tree depth (#126)

# What ❔

Implements dynamic depth of the in-memory L2-to-L1 log Merkle tree.
Previously, this tree always had 512 items (if necessary, additional
zero items were added at the end). With these changes, the tree has *at
least* 512 items (with padding); the actual number of items is `max(512,
items.len().next_power_of_two())`. This makes the change
backward-compatible without needing any logic tied to L1 batch number
etc.

## Why ❔

We want to allow larger Merkle tree depths than previously.

## Checklist

- [x] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [x] Tests for the changes have been added / updated.
- [x] Documentation comments have been added / updated.
- [x] Code has been formatted via `zk fmt` and `zk lint`.

core/lib/mini_merkle_tree/benches/tree.rs

@@ -10,7 +10,7 @@ const TREE_SIZES: &[usize] = &[32, 64, 128, 256, 512, 1_024];
 
 fn compute_merkle_root(bencher: &mut Bencher<'_>, tree_size: usize) {
     let leaves = (0..tree_size).map(|i| [i as u8; 88]);
-    let tree = MiniMerkleTree::new(leaves, tree_size);
+    let tree = MiniMerkleTree::new(leaves, None);
     bencher.iter_batched(
         || tree.clone(),
         MiniMerkleTree::merkle_root,
@@ -20,7 +20,7 @@ fn compute_merkle_root(bencher: &mut Bencher<'_>, tree_size: usize) {
 
 fn compute_merkle_path(bencher: &mut Bencher<'_>, tree_size: usize) {
     let leaves = (0..tree_size).map(|i| [i as u8; 88]);
-    let tree = MiniMerkleTree::new(leaves, tree_size);
+    let tree = MiniMerkleTree::new(leaves, None);
     bencher.iter_batched(
         || tree.clone(),
         |tree| tree.merkle_root_and_path(tree_size / 3),

core/lib/mini_merkle_tree/src/lib.rs

@@ -15,8 +15,9 @@ mod tests;
 use zksync_basic_types::H256;
 use zksync_crypto::hasher::{keccak::KeccakHasher, Hasher};
 
-/// Maximum supported depth of Merkle trees. 10 means that the tree must have <=1,024 leaves.
-const MAX_TREE_DEPTH: usize = 10;
+/// Maximum supported depth of the tree. 32 corresponds to `2^32` elements in the tree, which
+/// we unlikely to ever hit.
+const MAX_TREE_DEPTH: usize = 32;
 
 /// In-memory Merkle tree of bounded depth (no more than 10).
 ///
@@ -27,61 +28,61 @@ const MAX_TREE_DEPTH: usize = 10;
 pub struct MiniMerkleTree<'a, const LEAF_SIZE: usize> {
     hasher: &'a dyn HashEmptySubtree<LEAF_SIZE>,
     hashes: Box<[H256]>,
-    tree_size: usize,
+    binary_tree_size: usize,
 }
 
 impl<const LEAF_SIZE: usize> MiniMerkleTree<'static, LEAF_SIZE>
 where
     KeccakHasher: HashEmptySubtree<LEAF_SIZE>,
 {
-    /// Creates a new Merkle tree from the supplied leaves. If `tree_size` is larger than the
-    /// number of the supplied leaves, the remaining leaves are `[0_u8; LEAF_SIZE]`.
+    /// Creates a new Merkle tree from the supplied leaves. If `min_tree_size` is supplied and is larger
+    /// than the number of the supplied leaves, the leaves are padded to `min_tree_size` with `[0_u8; LEAF_SIZE]` entries.
     /// The hash function used in keccak-256.
     ///
     /// # Panics
     ///
     /// Panics in the same situations as [`Self::with_hasher()`].
-    pub fn new(leaves: impl Iterator<Item = [u8; LEAF_SIZE]>, tree_size: usize) -> Self {
-        Self::with_hasher(&KeccakHasher, leaves, tree_size)
+    pub fn new(
+        leaves: impl Iterator<Item = [u8; LEAF_SIZE]>,
+        min_tree_size: Option<usize>,
+    ) -> Self {
+        Self::with_hasher(&KeccakHasher, leaves, min_tree_size)
     }
 }
 
 impl<'a, const LEAF_SIZE: usize> MiniMerkleTree<'a, LEAF_SIZE> {
-    /// Creates a new Merkle tree from the supplied leaves. If `tree_size` is larger than the
-    /// number of the supplied leaves, the remaining leaves are `[0_u8; LEAF_SIZE]`.
+    /// Creates a new Merkle tree from the supplied leaves. If `min_tree_size` is supplied and is larger than the
+    /// number of the supplied leaves, the leaves are padded to `min_tree_size` with `[0_u8; LEAF_SIZE]` entries.
     ///
     /// # Panics
     ///
     /// Panics if any of the following conditions applies:
     ///
-    /// - The number of `leaves` is greater than `tree_size`.
-    /// - `tree_size > 1_024`.
-    /// - `tree_size` is not a power of 2.
+    /// - `min_tree_size` (if supplied) is not a power of 2.
     pub fn with_hasher(
         hasher: &'a dyn HashEmptySubtree<LEAF_SIZE>,
         leaves: impl Iterator<Item = [u8; LEAF_SIZE]>,
-        tree_size: usize,
+        min_tree_size: Option<usize>,
     ) -> Self {
-        assert!(
-            tree_size <= 1 << MAX_TREE_DEPTH,
-            "tree size must be <={}",
-            1 << MAX_TREE_DEPTH
-        );
-        assert!(
-            tree_size.is_power_of_two(),
-            "tree size must be a power of 2"
-        );
-
         let hashes: Box<[H256]> = leaves.map(|bytes| hasher.hash_bytes(&bytes)).collect();
+        let mut binary_tree_size = hashes.len().next_power_of_two();
+        if let Some(min_tree_size) = min_tree_size {
+            assert!(
+                min_tree_size.is_power_of_two(),
+                "tree size must be a power of 2"
+            );
+            binary_tree_size = min_tree_size.max(binary_tree_size);
+        }
         assert!(
-            hashes.len() <= tree_size,
-            "tree size must be greater or equal the number of supplied leaves"
+            tree_depth_by_size(binary_tree_size) <= MAX_TREE_DEPTH,
+            "Tree contains more than {} items; this is not supported",
+            1 << MAX_TREE_DEPTH
         );
 
         Self {
             hasher,
             hashes,
-            tree_size,
+            binary_tree_size,
         }
     }
 
@@ -97,7 +98,7 @@ impl<'a, const LEAF_SIZE: usize> MiniMerkleTree<'a, LEAF_SIZE> {
 
     /// Returns the root hash and the Merkle proof for a leaf with the specified 0-based `index`.
     pub fn merkle_root_and_path(self, index: usize) -> (H256, Vec<H256>) {
-        let mut merkle_path = Vec::with_capacity(MAX_TREE_DEPTH);
+        let mut merkle_path = vec![];
         let root_hash = self.compute_merkle_root_and_path(index, Some(&mut merkle_path));
         (root_hash, merkle_path)
     }
@@ -109,7 +110,10 @@ impl<'a, const LEAF_SIZE: usize> MiniMerkleTree<'a, LEAF_SIZE> {
     ) -> H256 {
         assert!(index < self.hashes.len(), "invalid tree leaf index");
 
-        let depth = tree_depth_by_size(self.tree_size);
+        let depth = tree_depth_by_size(self.binary_tree_size);
+        if let Some(merkle_path) = merkle_path.as_deref_mut() {
+            merkle_path.reserve(depth);
+        }
 
         let mut hashes = self.hashes;
         let mut level_len = hashes.len();

core/lib/mini_merkle_tree/src/tests.rs

@@ -26,7 +26,7 @@ fn hash_of_empty_tree_with_single_item() {
     for depth in 0..=5 {
         let len = 1 << depth;
         println!("checking tree with {len} items");
-        let tree = MiniMerkleTree::new(iter::once([0_u8; 88]), len);
+        let tree = MiniMerkleTree::new(iter::once([0_u8; 88]), Some(len));
         assert_eq!(tree.merkle_root(), KeccakHasher.empty_subtree_hash(depth));
     }
 }
@@ -38,16 +38,18 @@ fn hash_of_large_empty_tree_with_multiple_items() {
         let leaves = iter::repeat([0_u8; 88]).take(len);
         let tree_size = len.next_power_of_two();
 
-        let tree = MiniMerkleTree::new(leaves, tree_size);
+        let tree = MiniMerkleTree::new(leaves.clone(), Some(tree_size));
+        let depth = tree_depth_by_size(tree_size);
+        assert_eq!(tree.merkle_root(), KeccakHasher.empty_subtree_hash(depth));
+        let tree = MiniMerkleTree::new(leaves, None);
         let depth = tree_depth_by_size(tree_size);
-        assert!(depth <= MAX_TREE_DEPTH);
         assert_eq!(tree.merkle_root(), KeccakHasher.empty_subtree_hash(depth));
     }
 }
 
 #[test]
 fn single_item_tree_snapshot() {
-    let tree = MiniMerkleTree::new(iter::once([1_u8; 88]), 32);
+    let tree = MiniMerkleTree::new(iter::once([1_u8; 88]), Some(32));
     let (root_hash, path) = tree.merkle_root_and_path(0);
 
     let expected_root_hash: H256 =
@@ -70,7 +72,7 @@ fn single_item_tree_snapshot() {
 #[test]
 fn full_tree_snapshot() {
     let leaves = (1_u8..=32).map(|byte| [byte; 88]);
-    let tree = MiniMerkleTree::new(leaves, 32);
+    let tree = MiniMerkleTree::new(leaves, None);
     let (root_hash, path) = tree.merkle_root_and_path(2);
 
     let expected_root_hash: H256 =
@@ -93,7 +95,7 @@ fn full_tree_snapshot() {
 #[test]
 fn partial_tree_snapshot() {
     let leaves = (1_u8..=50).map(|byte| [byte; 88]);
-    let tree = MiniMerkleTree::new(leaves.clone(), 64);
+    let tree = MiniMerkleTree::new(leaves.clone(), None);
     let (root_hash, path) = tree.merkle_root_and_path(10);
 
     let expected_root_hash: H256 =
@@ -113,7 +115,7 @@ fn partial_tree_snapshot() {
     .map(|s| s.parse::<H256>().unwrap());
     assert_eq!(path, expected_path);
 
-    let tree = MiniMerkleTree::new(leaves, 64);
+    let tree = MiniMerkleTree::new(leaves, None);
     let (root_hash, path) = tree.merkle_root_and_path(49);
 
     assert_eq!(root_hash, expected_root_hash);
@@ -157,7 +159,7 @@ fn verify_merkle_proof(
 #[test]
 fn merkle_proofs_are_valid_in_small_tree() {
     let leaves = (1_u8..=50).map(|byte| [byte; 88]);
-    let tree = MiniMerkleTree::new(leaves.clone(), 64);
+    let tree = MiniMerkleTree::new(leaves.clone(), None);
 
     for (i, item) in leaves.enumerate() {
         let (merkle_root, path) = tree.clone().merkle_root_and_path(i);
@@ -168,10 +170,50 @@ fn merkle_proofs_are_valid_in_small_tree() {
 #[test]
 fn merkle_proofs_are_valid_in_larger_tree() {
     let leaves = (1_u8..=255).map(|byte| [byte; 88]);
-    let tree = MiniMerkleTree::new(leaves.clone(), 512);
+    let tree = MiniMerkleTree::new(leaves.clone(), Some(512));
 
     for (i, item) in leaves.enumerate() {
         let (merkle_root, path) = tree.clone().merkle_root_and_path(i);
         verify_merkle_proof(&item, i, 512, &path, merkle_root);
     }
 }
+
+#[test]
+#[allow(clippy::cast_possible_truncation)] // truncation is intentional
+fn merkle_proofs_are_valid_in_very_large_tree() {
+    let leaves = (1_u32..=15_000).map(|byte| [byte as u8; 88]);
+
+    let tree = MiniMerkleTree::new(leaves.clone(), None);
+    for (i, item) in leaves.clone().enumerate().step_by(61) {
+        let (merkle_root, path) = tree.clone().merkle_root_and_path(i);
+        verify_merkle_proof(&item, i, 1 << 14, &path, merkle_root);
+    }
+
+    let tree_with_min_size = MiniMerkleTree::new(leaves.clone(), Some(512));
+    assert_eq!(tree_with_min_size.clone().merkle_root(), tree.merkle_root());
+    for (i, item) in leaves.enumerate().step_by(61) {
+        let (merkle_root, path) = tree_with_min_size.clone().merkle_root_and_path(i);
+        verify_merkle_proof(&item, i, 1 << 14, &path, merkle_root);
+    }
+}
+
+#[test]
+fn merkle_proofs_are_valid_in_very_small_trees() {
+    for item_count in 1..=20 {
+        let leaves = (1..=item_count).map(|byte| [byte; 88]);
+
+        let tree = MiniMerkleTree::new(leaves.clone(), None);
+        let item_count = usize::from(item_count).next_power_of_two();
+        for (i, item) in leaves.clone().enumerate() {
+            let (merkle_root, path) = tree.clone().merkle_root_and_path(i);
+            verify_merkle_proof(&item, i, item_count, &path, merkle_root);
+        }
+
+        let tree_with_min_size = MiniMerkleTree::new(leaves.clone(), Some(512));
+        assert_ne!(tree_with_min_size.clone().merkle_root(), tree.merkle_root());
+        for (i, item) in leaves.enumerate() {
+            let (merkle_root, path) = tree_with_min_size.clone().merkle_root_and_path(i);
+            verify_merkle_proof(&item, i, 512, &path, merkle_root);
+        }
+    }
+}

core/lib/types/src/commitment.rs

@@ -15,7 +15,6 @@ use zksync_mini_merkle_tree::MiniMerkleTree;
 
 use crate::{
     block::L1BatchHeader,
-    circuit::GEOMETRY_CONFIG,
     ethabi::Token,
     l2_to_l1_log::L2ToL1Log,
     web3::signing::keccak256,
@@ -27,8 +26,6 @@ use crate::{
 pub trait SerializeCommitment {
     /// Size of the structure in bytes.
     const SERIALIZED_SIZE: usize;
-    /// The number of objects of this type that can be included in a single L1 batch.
-    const LIMIT_PER_L1_BATCH: usize;
     /// Serializes this struct into the provided buffer, which is guaranteed to have byte length
     /// [`Self::SERIALIZED_SIZE`].
     fn serialize_commitment(&self, buffer: &mut [u8]);
@@ -167,7 +164,6 @@ impl L1BatchWithMetadata {
 
 impl SerializeCommitment for L2ToL1Log {
     const SERIALIZED_SIZE: usize = 88;
-    const LIMIT_PER_L1_BATCH: usize = GEOMETRY_CONFIG.limit_for_l1_messages_merklizer as usize;
 
     fn serialize_commitment(&self, buffer: &mut [u8]) {
         buffer[0] = self.shard_id;
@@ -181,8 +177,6 @@ impl SerializeCommitment for L2ToL1Log {
 
 impl SerializeCommitment for InitialStorageWrite {
     const SERIALIZED_SIZE: usize = 64;
-    const LIMIT_PER_L1_BATCH: usize =
-        GEOMETRY_CONFIG.limit_for_initial_writes_pubdata_hasher as usize;
 
     fn serialize_commitment(&self, buffer: &mut [u8]) {
         self.key.to_little_endian(&mut buffer[0..32]);
@@ -192,8 +186,6 @@ impl SerializeCommitment for InitialStorageWrite {
 
 impl SerializeCommitment for RepeatedStorageWrite {
     const SERIALIZED_SIZE: usize = 40;
-    const LIMIT_PER_L1_BATCH: usize =
-        GEOMETRY_CONFIG.limit_for_repeated_writes_pubdata_hasher as usize;
 
     fn serialize_commitment(&self, buffer: &mut [u8]) {
         buffer[..8].copy_from_slice(&self.index.to_be_bytes());
@@ -238,8 +230,9 @@ impl L1BatchAuxiliaryOutput {
             .chunks(L2ToL1Log::SERIALIZED_SIZE)
             .map(|chunk| <[u8; L2ToL1Log::SERIALIZED_SIZE]>::try_from(chunk).unwrap());
         // ^ Skip first 4 bytes of the serialized logs (i.e., the number of logs).
+        let min_tree_size = Some(L2ToL1Log::LEGACY_LIMIT_PER_L1_BATCH);
         let l2_l1_logs_merkle_root =
-            MiniMerkleTree::new(merkle_tree_leaves, L2ToL1Log::LIMIT_PER_L1_BATCH).merkle_root();
+            MiniMerkleTree::new(merkle_tree_leaves, min_tree_size).merkle_root();
 
         Self {
             l2_l1_logs_compressed,

core/lib/types/src/l2_to_l1_log.rs

@@ -15,6 +15,11 @@ pub struct L2ToL1Log {
 }
 
 impl L2ToL1Log {
+    /// Legacy upper bound of L2-to-L1 logs per single L1 batch. This is not used as a limit now,
+    /// but still determines the minimum number of items in the Merkle tree built from L2-to-L1 logs
+    /// for a certain batch.
+    pub const LEGACY_LIMIT_PER_L1_BATCH: usize = 512;
+
     pub fn from_slice(data: &[u8]) -> Self {
         assert_eq!(data.len(), Self::SERIALIZED_SIZE);
         Self {