ZcashFoundation · mergify · Jun 30, 2022 · Jun 16, 2022 · Jun 16, 2022 · Jun 17, 2022
@@ -422,16 +422,3 @@ cleaner and the cost is probably not too large.
 - We need to pick a timeout for UTXO lookup. This should be long enough to
 account for the fact that we may start verifying blocks before all of their
 ancestors are downloaded.
-
-These optimisations can be delayed until after the initial implementation is
-complete, and covered by tests:
-
-- Should we stop storing heights for non-coinbase UTXOs? (#2455)
-
-- Should we avoid storing any extra data for UTXOs, and just lookup the coinbase
-  flag and height using `outpoint.hash` and `tx_by_hash`? (#2455)
-
-- The maturity check can be skipped for UTXOs from the finalized state,
-because Zebra only finalizes mature UTXOs. We could implement this
-optimisation by adding a `Utxo::MatureCoinbase { output: transparent::Output }`
-variant, which only performs the spend checks. (#2455)
@@ -268,20 +268,90 @@ is completely empty.
 The `Chain` type is defined by the following struct and API:
 
 ```rust
-#[derive(Debug, Default, Clone)]
-struct Chain {
-    blocks: BTreeMap<block::Height, Arc<Block>>,
-    height_by_hash: HashMap<block::Hash, block::Height>,
-    tx_by_hash: HashMap<transaction::Hash, (block::Height, usize)>,
-
-    created_utxos: HashSet<transparent::OutPoint>,
-    spent_utxos: HashSet<transparent::OutPoint>,
-    sprout_anchors: HashSet<sprout::tree::Root>,
-    sapling_anchors: HashSet<sapling::tree::Root>,
-    sprout_nullifiers: HashSet<sprout::Nullifier>,
-    sapling_nullifiers: HashSet<sapling::Nullifier>,
-    orchard_nullifiers: HashSet<orchard::Nullifier>,
-    partial_cumulative_work: PartialCumulativeWork,
+#[derive(Debug, Clone)]
+pub struct Chain {
+    // The function `eq_internal_state` must be updated every time a field is added to [`Chain`].
+    /// The configured network for this chain.
+    network: Network,
+
+    /// The contextually valid blocks which form this non-finalized partial chain, in height order.
+    pub(crate) blocks: BTreeMap<block::Height, ContextuallyValidBlock>,
+
+    /// An index of block heights for each block hash in `blocks`.
+    pub height_by_hash: HashMap<block::Hash, block::Height>,
+
+    /// An index of [`TransactionLocation`]s for each transaction hash in `blocks`.
+    pub tx_loc_by_hash: HashMap<transaction::Hash, TransactionLocation>,
+
+    /// The [`transparent::Utxo`]s created by `blocks`.
+    ///
+    /// Note that these UTXOs may not be unspent.
+    /// Outputs can be spent by later transactions or blocks in the chain.
+    //
+    // TODO: replace OutPoint with OutputLocation?
+    pub(crate) created_utxos: HashMap<transparent::OutPoint, transparent::OrderedUtxo>,
+    /// The [`transparent::OutPoint`]s spent by `blocks`,
+    /// including those created by earlier transactions or blocks in the chain.
+    pub(crate) spent_utxos: HashSet<transparent::OutPoint>,
+
+    /// The Sprout note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) sprout_note_commitment_tree: sprout::tree::NoteCommitmentTree,
+    /// The Sprout note commitment tree for each anchor.
+    /// This is required for interstitial states.
+    pub(crate) sprout_trees_by_anchor:
+        HashMap<sprout::tree::Root, sprout::tree::NoteCommitmentTree>,
+    /// The Sapling note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree,
+    /// The Sapling note commitment tree for each height.
+    pub(crate) sapling_trees_by_height: BTreeMap<block::Height, sapling::tree::NoteCommitmentTree>,
+    /// The Orchard note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree,
+    /// The Orchard note commitment tree for each height.
+    pub(crate) orchard_trees_by_height: BTreeMap<block::Height, orchard::tree::NoteCommitmentTree>,
+    /// The ZIP-221 history tree of the tip of this [`Chain`],
+    /// including all finalized blocks, and the non-finalized `blocks` in this chain.
+    pub(crate) history_tree: HistoryTree,
+
+    /// The Sprout anchors created by `blocks`.
+    pub(crate) sprout_anchors: MultiSet<sprout::tree::Root>,
+    /// The Sprout anchors created by each block in `blocks`.
+    pub(crate) sprout_anchors_by_height: BTreeMap<block::Height, sprout::tree::Root>,
+    /// The Sapling anchors created by `blocks`.
+    pub(crate) sapling_anchors: MultiSet<sapling::tree::Root>,
+    /// The Sapling anchors created by each block in `blocks`.
+    pub(crate) sapling_anchors_by_height: BTreeMap<block::Height, sapling::tree::Root>,
+    /// The Orchard anchors created by `blocks`.
+    pub(crate) orchard_anchors: MultiSet<orchard::tree::Root>,
+    /// The Orchard anchors created by each block in `blocks`.
+    pub(crate) orchard_anchors_by_height: BTreeMap<block::Height, orchard::tree::Root>,
+
+    /// The Sprout nullifiers revealed by `blocks`.
+    pub(super) sprout_nullifiers: HashSet<sprout::Nullifier>,
+    /// The Sapling nullifiers revealed by `blocks`.
+    pub(super) sapling_nullifiers: HashSet<sapling::Nullifier>,
+    /// The Orchard nullifiers revealed by `blocks`.
+    pub(super) orchard_nullifiers: HashSet<orchard::Nullifier>,
+
+    /// Partial transparent address index data from `blocks`.
+    pub(super) partial_transparent_transfers: HashMap<transparent::Address, TransparentTransfers>,
+
+    /// The cumulative work represented by `blocks`.
+    ///
+    /// Since the best chain is determined by the largest cumulative work,
+    /// the work represented by finalized blocks can be ignored,
+    /// because they are common to all non-finalized chains.
+    pub(super) partial_cumulative_work: PartialCumulativeWork,
+
+    /// The chain value pool balances of the tip of this [`Chain`],
+    /// including the block value pool changes from all finalized blocks,
+    /// and the non-finalized blocks in this chain.
+    ///
+    /// When a new chain is created from the finalized tip,
+    /// it is initialized with the finalized tip chain value pool balances.
+    pub(crate) chain_value_pools: ValueBalance<NonNegative>,
 }
 ```
 
@@ -293,7 +363,7 @@ Push a block into a chain as the new tip
     - Add the block's hash to `height_by_hash`
     - Add work to `self.partial_cumulative_work`
     - For each `transaction` in `block`
-      - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_by_hash`
+      - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_loc_by_hash`
       - Add created utxos to `self.created_utxos`
       - Add spent utxos to `self.spent_utxos`
       - Add nullifiers to the appropriate `self.<version>_nullifiers`
@@ -310,7 +380,7 @@ Remove the lowest height block of the non-finalized portion of a chain.
     - Remove the block's hash from `self.height_by_hash`
     - Subtract work from `self.partial_cumulative_work`
     - For each `transaction` in `block`
-      - Remove `transaction.hash` from `tx_by_hash`
+      - Remove `transaction.hash` from `tx_loc_by_hash`
       - Remove created utxos from `self.created_utxos`
       - Remove spent utxos from `self.spent_utxos`
       - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
@@ -340,7 +410,7 @@ Remove the highest height block of the non-finalized portion of a chain.
     - Remove the corresponding hash from `self.height_by_hash`
     - Subtract work from `self.partial_cumulative_work`
     - for each `transaction` in `block`
-      - remove `transaction.hash` from `tx_by_hash`
+      - remove `transaction.hash` from `tx_loc_by_hash`
       - Remove created utxos from `self.created_utxos`
       - Remove spent utxos from `self.spent_utxos`
       - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
@@ -365,7 +435,7 @@ parent block is the tip of the finalized state. This implementation should be
 handled by `#[derive(Default)]`.
 
 1. initialise cumulative data members
-    - Construct an empty `self.blocks`, `height_by_hash`, `tx_by_hash`,
+    - Construct an empty `self.blocks`, `height_by_hash`, `tx_loc_by_hash`,
     `self.created_utxos`, `self.spent_utxos`, `self.<version>_anchors`,
     `self.<version>_nullifiers`
     - Zero `self.partial_cumulative_work`
@@ -1102,13 +1172,14 @@ Returns
 
 Implemented by querying:
 
-- (non-finalized) the `tx_by_hash` map (to get the block that contains the
+- (non-finalized) the `tx_loc_by_hash` map (to get the block that contains the
   transaction) of each chain starting with the best chain, and then find
   block that chain's `blocks` (to get the block containing the transaction
   data)
-- (finalized) the `tx_by_hash` tree (to get the block that contains the
-  transaction) and then `block_by_height` tree (to get the block containing
-  the transaction data), if the transaction is not in any non-finalized chain
+- (finalized) the `tx_loc_by_hash` tree (to get the block that contains the
+  transaction) and then `block_header_by_height` tree (to get the block
+  containing the transaction data), if the transaction is not in any
+  non-finalized chain
 
 ### `Request::Block(block::Hash)`
 [request-block]: #request-block
@@ -1125,8 +1196,9 @@ Implemented by querying:
 
 - (non-finalized) the `height_by_hash` of each chain starting with the best
   chain, then find block that chain's `blocks` (to get the block data)
-- (finalized) the `height_by_hash` tree (to get the block height) and then
-    the `block_by_height` tree (to get the block data), if the block is not in any non-finalized chain
+- (finalized) the `height_by_hash` tree (to get the block height) and then the
+  `block_header_by_height` tree (to get the block data), if the block is not in
+  any non-finalized chain
 
 ### `Request::AwaitSpendableUtxo { outpoint: OutPoint, spend_height: Height, spend_restriction: SpendRestriction }`
 

@@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
-pub const DATABASE_FORMAT_VERSION: u32 = 24;
+pub const DATABASE_FORMAT_VERSION: u32 = 25;
 
 /// The maximum number of blocks to check for NU5 transactions,
 /// before we assume we are on a pre-NU5 legacy chain.

@@ -369,27 +369,22 @@ impl DiskDb {
 
         let column_families = vec![
             // Blocks
-            // TODO: rename to block_header_by_height (#3151)
-            rocksdb::ColumnFamilyDescriptor::new("block_by_height", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("hash_by_height", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("height_by_hash", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("block_header_by_height", db_options.clone()),
             // Transactions
             rocksdb::ColumnFamilyDescriptor::new("tx_by_loc", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("hash_by_tx_loc", db_options.clone()),
-            // TODO: rename to tx_loc_by_hash (#3950)
-            rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("tx_loc_by_hash", db_options.clone()),
             // Transparent
             rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()),
-            // TODO: #3951
-            //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()),
-            // TODO: rename to utxo_by_out_loc (#3952)
-            rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new(
-                "utxo_loc_by_transparent_addr_loc",
+                "tx_loc_by_transparent_addr_loc",
                 db_options.clone(),
             ),
+            rocksdb::ColumnFamilyDescriptor::new("utxo_by_out_loc", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new(
-                "tx_loc_by_transparent_addr_loc",
+                "utxo_loc_by_transparent_addr_loc",
                 db_options.clone(),
             ),
             // Sprout

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,10 +1,11 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 72
 expression: cf_names
 ---
 [
   "balance_by_transparent_addr",
-  "block_by_height",
+  "block_header_by_height",
   "default",
   "hash_by_height",
   "hash_by_tx_loc",
@@ -20,9 +21,9 @@ expression: cf_names
   "sprout_note_commitment_tree",
   "sprout_nullifiers",
   "tip_chain_value_pool",
-  "tx_by_hash",
   "tx_by_loc",
+  "tx_loc_by_hash",
   "tx_loc_by_transparent_addr_loc",
-  "utxo_by_outpoint",
+  "utxo_by_out_loc",
   "utxo_loc_by_transparent_addr_loc",
 ]
@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 154
 expression: empty_column_families
 ---
 [
@@ -13,6 +14,6 @@ expression: empty_column_families
   "sprout_nullifiers: no entries",
   "tip_chain_value_pool: no entries",
   "tx_loc_by_transparent_addr_loc: no entries",
-  "utxo_by_outpoint: no entries",
+  "utxo_by_out_loc: no entries",
   "utxo_loc_by_transparent_addr_loc: no entries",
 ]
@@ -1,10 +1,11 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 154
 expression: empty_column_families
 ---
 [
   "balance_by_transparent_addr: no entries",
-  "block_by_height: no entries",
+  "block_header_by_height: no entries",
   "hash_by_height: no entries",
   "hash_by_tx_loc: no entries",
   "height_by_hash: no entries",
@@ -19,9 +20,9 @@ expression: empty_column_families
   "sprout_note_commitment_tree: no entries",
   "sprout_nullifiers: no entries",
   "tip_chain_value_pool: no entries",
-  "tx_by_hash: no entries",
   "tx_by_loc: no entries",
+  "tx_loc_by_hash: no entries",
   "tx_loc_by_transparent_addr_loc: no entries",
-  "utxo_by_outpoint: no entries",
+  "utxo_by_out_loc: no entries",
   "utxo_loc_by_transparent_addr_loc: no entries",
 ]
@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 154
 expression: empty_column_families
 ---
 [
@@ -13,6 +14,6 @@ expression: empty_column_families
   "sprout_nullifiers: no entries",
   "tip_chain_value_pool: no entries",
   "tx_loc_by_transparent_addr_loc: no entries",
-  "utxo_by_outpoint: no entries",
+  "utxo_by_out_loc: no entries",
   "utxo_loc_by_transparent_addr_loc: no entries",
 ]
@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [

@@ -1,5 +1,6 @@
 ---
 source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
 expression: cf_data
 ---
 [