change(state): Insert only the first tree in each series of identical…

… trees into finalized state (#7266) * Pass ZebraDB to batch preparation * Dedup the insertion of Sapling trees into database * Dedup the insertion of Orchard trees into database * Update snapshots * Rename batch preparation of trees * Simplify the naming of note commitment trees * Correctly retrieve Sapling trees from fin state * Correctly retrieve Orchard trees from fin state * Simplify the naming of methods for Sprout trees * Simplify the naming of methods for Sapling trees * Simplify the naming of methods for Orchard trees * Reduce disk reads by caching trees. (#7276) * Bump the state minor version * Reset the state patch version * Simplify the preparation of genesis trees * Store the roots of the trees of the genesis block * Add the genesis roots to snapshots * fix(test): Don't include shielded data in genesis blocks (#7302) * fix(state): Fix marking format upgrades (#7304) --------- Co-authored-by: Arya <[email protected]>
ZcashFoundation · Aug 9, 2023 · 57c9249 · 57c9249
1 parent cce81b3
commit 57c9249
Show file tree

Hide file tree

Showing 38 changed files with 259 additions and 239 deletions.
diff --git a/zebra-chain/src/transaction/arbitrary.rs b/zebra-chain/src/transaction/arbitrary.rs
@@ -109,20 +109,32 @@ impl Transaction {
             option::of(any::<sapling::ShieldedData<sapling::PerSpendAnchor>>()),
         )
             .prop_map(
-                |(
-                    inputs,
-                    outputs,
-                    lock_time,
-                    expiry_height,
-                    joinsplit_data,
-                    sapling_shielded_data,
-                )| Transaction::V4 {
+                move |(
                     inputs,
                     outputs,
                     lock_time,
                     expiry_height,
                     joinsplit_data,
                     sapling_shielded_data,
+                )| {
+                    Transaction::V4 {
+                        inputs,
+                        outputs,
+                        lock_time,
+                        expiry_height,
+                        joinsplit_data: if ledger_state.height.is_min() {
+                            // The genesis block should not contain any joinsplits.
+                            None
+                        } else {
+                            joinsplit_data
+                        },
+                        sapling_shielded_data: if ledger_state.height.is_min() {
+                            // The genesis block should not contain any shielded data.
+                            None
+                        } else {
+                            sapling_shielded_data
+                        },
+                    }
                 },
             )
             .boxed()
@@ -159,8 +171,18 @@ impl Transaction {
                         expiry_height,
                         inputs,
                         outputs,
-                        sapling_shielded_data,
-                        orchard_shielded_data,
+                        sapling_shielded_data: if ledger_state.height.is_min() {
+                            // The genesis block should not contain any shielded data.
+                            None
+                        } else {
+                            sapling_shielded_data
+                        },
+                        orchard_shielded_data: if ledger_state.height.is_min() {
+                            // The genesis block should not contain any shielded data.
+                            None
+                        } else {
+                            orchard_shielded_data
+                        },
                     }
                 },
             )

diff --git a/zebra-state/src/constants.rs b/zebra-state/src/constants.rs
@@ -48,11 +48,11 @@ pub(crate) const DATABASE_FORMAT_VERSION: u64 = 25;
 /// - adding new column families,
 /// - changing the format of a column family in a compatible way, or
 /// - breaking changes with compatibility code in all supported Zebra versions.
-pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 0;
+pub(crate) const DATABASE_FORMAT_MINOR_VERSION: u64 = 1;
 
 /// The database format patch version, incremented each time the on-disk database format has a
 /// significant format compatibility fix.
-pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 2;
+pub(crate) const DATABASE_FORMAT_PATCH_VERSION: u64 = 0;
 
 /// The name of the file containing the minor and patch database versions.
 ///

diff --git a/zebra-state/src/service/check/anchors.rs b/zebra-state/src/service/check/anchors.rs
@@ -152,7 +152,7 @@ fn fetch_sprout_final_treestates(
 
         let input_tree = parent_chain
             .and_then(|chain| chain.sprout_trees_by_anchor.get(&joinsplit.anchor).cloned())
-            .or_else(|| finalized_state.sprout_note_commitment_tree_by_anchor(&joinsplit.anchor));
+            .or_else(|| finalized_state.sprout_tree_by_anchor(&joinsplit.anchor));
 
         if let Some(input_tree) = input_tree {
             sprout_final_treestates.insert(joinsplit.anchor, input_tree);

diff --git a/zebra-state/src/service/check/tests/nullifier.rs b/zebra-state/src/service/check/tests/nullifier.rs
@@ -85,7 +85,7 @@ proptest! {
         // randomly choose to commit the block to the finalized or non-finalized state
         if use_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
             // the block was committed
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@@ -352,7 +352,7 @@ proptest! {
         // randomly choose to commit the next block to the finalized or non-finalized state
         if duplicate_in_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
             prop_assert!(commit_result.is_ok());
@@ -452,7 +452,7 @@ proptest! {
         // randomly choose to commit the block to the finalized or non-finalized state
         if use_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None,  "test");
 
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
             prop_assert!(commit_result.is_ok());
@@ -634,7 +634,7 @@ proptest! {
         // randomly choose to commit the next block to the finalized or non-finalized state
         if duplicate_in_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(),None,  "test");
 
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
             prop_assert!(commit_result.is_ok());
@@ -732,7 +732,7 @@ proptest! {
         // randomly choose to commit the block to the finalized or non-finalized state
         if use_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
             prop_assert!(commit_result.is_ok());
@@ -923,7 +923,7 @@ proptest! {
         // randomly choose to commit the next block to the finalized or non-finalized state
         if duplicate_in_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
             prop_assert!(commit_result.is_ok());

diff --git a/zebra-state/src/service/check/tests/utxo.rs b/zebra-state/src/service/check/tests/utxo.rs
@@ -185,7 +185,7 @@ proptest! {
         // randomly choose to commit the block to the finalized or non-finalized state
         if use_finalized_state {
             let block1 = CheckpointVerifiedBlock::from(Arc::new(block1));
-            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
             // the block was committed
             prop_assert_eq!(Some((Height(1), block1.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@@ -273,7 +273,7 @@ proptest! {
 
         if use_finalized_state_spend {
             let block2 = CheckpointVerifiedBlock::from(Arc::new(block2));
-            let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(),None,  "test");
 
             // the block was committed
             prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@@ -612,7 +612,7 @@ proptest! {
 
         if use_finalized_state_spend {
             let block2 = CheckpointVerifiedBlock::from(block2.clone());
-            let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), "test");
+            let commit_result = finalized_state.commit_finalized_direct(block2.clone().into(), None, "test");
 
             // the block was committed
             prop_assert_eq!(Some((Height(2), block2.hash)), read::best_tip(&non_finalized_state, &finalized_state.db));
@@ -884,7 +884,8 @@ fn new_state_with_mainnet_transparent_data(
 
     if use_finalized_state {
         let block1 = CheckpointVerifiedBlock::from(block1.clone());
-        let commit_result = finalized_state.commit_finalized_direct(block1.clone().into(), "test");
+        let commit_result =
+            finalized_state.commit_finalized_direct(block1.clone().into(), None, "test");
 
         // the block was committed
         assert_eq!(

diff --git a/zebra-state/src/service/finalized_state.rs b/zebra-state/src/service/finalized_state.rs
@@ -20,7 +20,7 @@ use std::{
     sync::Arc,
 };
 
-use zebra_chain::{block, parameters::Network};
+use zebra_chain::{block, parallel::tree::NoteCommitmentTrees, parameters::Network};
 
 use crate::{
     request::{FinalizableBlock, SemanticallyVerifiedBlockWithTrees, Treestate},
@@ -168,10 +168,12 @@ impl FinalizedState {
     pub fn commit_finalized(
         &mut self,
         ordered_block: QueuedCheckpointVerified,
-    ) -> Result<CheckpointVerifiedBlock, BoxError> {
+        prev_note_commitment_trees: Option<NoteCommitmentTrees>,
+    ) -> Result<(CheckpointVerifiedBlock, NoteCommitmentTrees), BoxError> {
         let (checkpoint_verified, rsp_tx) = ordered_block;
         let result = self.commit_finalized_direct(
             checkpoint_verified.clone().into(),
+            prev_note_commitment_trees,
             "commit checkpoint-verified request",
         );
 
@@ -202,10 +204,10 @@ impl FinalizedState {
         // and the block write task.
         let result = result.map_err(CloneError::from);
 
-        let _ = rsp_tx.send(result.clone().map_err(BoxError::from));
+        let _ = rsp_tx.send(result.clone().map(|(hash, _)| hash).map_err(BoxError::from));
 
         result
-            .map(|_hash| checkpoint_verified)
+            .map(|(_hash, note_commitment_trees)| (checkpoint_verified, note_commitment_trees))
             .map_err(BoxError::from)
     }
 
@@ -226,9 +228,10 @@ impl FinalizedState {
     pub fn commit_finalized_direct(
         &mut self,
         finalizable_block: FinalizableBlock,
+        prev_note_commitment_trees: Option<NoteCommitmentTrees>,
         source: &str,
-    ) -> Result<block::Hash, BoxError> {
-        let (height, hash, finalized) = match finalizable_block {
+    ) -> Result<(block::Hash, NoteCommitmentTrees), BoxError> {
+        let (height, hash, finalized, prev_note_commitment_trees) = match finalizable_block {
             FinalizableBlock::Checkpoint {
                 checkpoint_verified,
             } => {
@@ -240,9 +243,11 @@ impl FinalizedState {
 
                 let block = checkpoint_verified.block.clone();
                 let mut history_tree = self.db.history_tree();
-                let mut note_commitment_trees = self.db.note_commitment_trees();
+                let prev_note_commitment_trees =
+                    prev_note_commitment_trees.unwrap_or_else(|| self.db.note_commitment_trees());
 
                 // Update the note commitment trees.
+                let mut note_commitment_trees = prev_note_commitment_trees.clone();
                 note_commitment_trees.update_trees_parallel(&block)?;
 
                 // Check the block commitment if the history tree was not
@@ -287,6 +292,7 @@ impl FinalizedState {
                             history_tree,
                         },
                     },
+                    Some(prev_note_commitment_trees),
                 )
             }
             FinalizableBlock::Contextual {
@@ -299,6 +305,7 @@ impl FinalizedState {
                     verified: contextually_verified.into(),
                     treestate,
                 },
+                prev_note_commitment_trees,
             ),
         };
 
@@ -331,8 +338,11 @@ impl FinalizedState {
 
         #[cfg(feature = "elasticsearch")]
         let finalized_block = finalized.verified.block.clone();
+        let note_commitment_trees = finalized.treestate.note_commitment_trees.clone();
 
-        let result = self.db.write_block(finalized, self.network, source);
+        let result =
+            self.db
+                .write_block(finalized, prev_note_commitment_trees, self.network, source);
 
         if result.is_ok() {
             // Save blocks to elasticsearch if the feature is enabled.
@@ -360,7 +370,7 @@ impl FinalizedState {
             }
         }
 
-        result
+        result.map(|hash| (hash, note_commitment_trees))
     }
 
     #[cfg(feature = "elasticsearch")]

diff --git a/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs b/zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
@@ -103,7 +103,7 @@ fn test_raw_rocksdb_column_families_with_network(network: Network) {
             .expect("test data deserializes");
 
         state
-            .commit_finalized_direct(block.into(), "snapshot tests")
+            .commit_finalized_direct(block.into(), None, "snapshot tests")
             .expect("test block is valid");
 
         let mut settings = insta::Settings::clone_current();

diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_0.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_0.snap
@@ -5,6 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
+    v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_1.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_1.snap
@@ -5,10 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
     v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_2.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@mainnet_2.snap
@@ -5,14 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
-    v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
-  ),
-  KV(
-    k: "000002",
     v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_0.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_0.snap
@@ -5,6 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
+    v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_1.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_1.snap
@@ -5,10 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
     v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_2.snap b/...ed_state/disk_format/tests/snapshots/orchard_note_commitment_tree_raw_data@testnet_2.snap
@@ -5,14 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
-    v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
-  ),
-  KV(
-    k: "000002",
     v: "0001ae2935f1dfd8a24aed7c70df7de3a668eb7a49b1319880dde2bbd9031ae5d82f",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_0.snap b/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_0.snap
@@ -5,6 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
+    v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_1.snap b/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_1.snap
@@ -5,10 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
     v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_2.snap b/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@mainnet_2.snap
@@ -5,14 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
-  ),
-  KV(
-    k: "000001",
-    v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
-  ),
-  KV(
-    k: "000002",
     v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
   ),
 ]
diff --git a/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@testnet_0.snap b/...ed_state/disk_format/tests/snapshots/sapling_note_commitment_tree_raw_data@testnet_0.snap
@@ -5,6 +5,6 @@ expression: cf_data
 [
   KV(
     k: "000000",
-    v: "0000",
+    v: "0001fbc2f4300c01f0b7820d00e3347c8da4ee614674376cbc45359daa54f9b5493e",
   ),
 ]