fix(state): Update column family names to match Zebra's database design

book/src/dev/rfcs/0004-asynchronous-script-verification.md

@@ -426,10 +426,12 @@ ancestors are downloaded.
 These optimisations can be delayed until after the initial implementation is
 complete, and covered by tests:
 
+<!-- TODO: address the three items below since #2455 is closed. -->
+
 - Should we stop storing heights for non-coinbase UTXOs? (#2455)
 
 - Should we avoid storing any extra data for UTXOs, and just lookup the coinbase
-  flag and height using `outpoint.hash` and `tx_by_hash`? (#2455)
+  flag and height using `outpoint.hash` and `tx_loc_by_hash`? (#2455)
 
 - The maturity check can be skipped for UTXOs from the finalized state,
 because Zebra only finalizes mature UTXOs. We could implement this

book/src/dev/rfcs/0005-state-updates.md

@@ -268,20 +268,90 @@ is completely empty.
 The `Chain` type is defined by the following struct and API:
 
 ```rust
-#[derive(Debug, Default, Clone)]
-struct Chain {
-    blocks: BTreeMap<block::Height, Arc<Block>>,
-    height_by_hash: HashMap<block::Hash, block::Height>,
-    tx_by_hash: HashMap<transaction::Hash, (block::Height, usize)>,
-
-    created_utxos: HashSet<transparent::OutPoint>,
-    spent_utxos: HashSet<transparent::OutPoint>,
-    sprout_anchors: HashSet<sprout::tree::Root>,
-    sapling_anchors: HashSet<sapling::tree::Root>,
-    sprout_nullifiers: HashSet<sprout::Nullifier>,
-    sapling_nullifiers: HashSet<sapling::Nullifier>,
-    orchard_nullifiers: HashSet<orchard::Nullifier>,
-    partial_cumulative_work: PartialCumulativeWork,
+#[derive(Debug, Clone)]
+pub struct Chain {
+    // The function `eq_internal_state` must be updated every time a field is added to [`Chain`].
+    /// The configured network for this chain.
+    network: Network,
+
+    /// The contextually valid blocks which form this non-finalized partial chain, in height order.
+    pub(crate) blocks: BTreeMap<block::Height, ContextuallyValidBlock>,
+
+    /// An index of block heights for each block hash in `blocks`.
+    pub height_by_hash: HashMap<block::Hash, block::Height>,
+
+    /// An index of [`TransactionLocation`]s for each transaction hash in `blocks`.
+    pub tx_loc_by_hash: HashMap<transaction::Hash, TransactionLocation>,
+
+    /// The [`transparent::Utxo`]s created by `blocks`.
+    ///
+    /// Note that these UTXOs may not be unspent.
+    /// Outputs can be spent by later transactions or blocks in the chain.
+    //
+    // TODO: replace OutPoint with OutputLocation?
+    pub(crate) created_utxos: HashMap<transparent::OutPoint, transparent::OrderedUtxo>,
+    /// The [`transparent::OutPoint`]s spent by `blocks`,
+    /// including those created by earlier transactions or blocks in the chain.
+    pub(crate) spent_utxos: HashSet<transparent::OutPoint>,
+
+    /// The Sprout note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) sprout_note_commitment_tree: sprout::tree::NoteCommitmentTree,
+    /// The Sprout note commitment tree for each anchor.
+    /// This is required for interstitial states.
+    pub(crate) sprout_trees_by_anchor:
+        HashMap<sprout::tree::Root, sprout::tree::NoteCommitmentTree>,
+    /// The Sapling note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) sapling_note_commitment_tree: sapling::tree::NoteCommitmentTree,
+    /// The Sapling note commitment tree for each height.
+    pub(crate) sapling_trees_by_height: BTreeMap<block::Height, sapling::tree::NoteCommitmentTree>,
+    /// The Orchard note commitment tree of the tip of this [`Chain`],
+    /// including all finalized notes, and the non-finalized notes in this chain.
+    pub(super) orchard_note_commitment_tree: orchard::tree::NoteCommitmentTree,
+    /// The Orchard note commitment tree for each height.
+    pub(crate) orchard_trees_by_height: BTreeMap<block::Height, orchard::tree::NoteCommitmentTree>,
+    /// The ZIP-221 history tree of the tip of this [`Chain`],
+    /// including all finalized blocks, and the non-finalized `blocks` in this chain.
+    pub(crate) history_tree: HistoryTree,
+
+    /// The Sprout anchors created by `blocks`.
+    pub(crate) sprout_anchors: MultiSet<sprout::tree::Root>,
+    /// The Sprout anchors created by each block in `blocks`.
+    pub(crate) sprout_anchors_by_height: BTreeMap<block::Height, sprout::tree::Root>,
+    /// The Sapling anchors created by `blocks`.
+    pub(crate) sapling_anchors: MultiSet<sapling::tree::Root>,
+    /// The Sapling anchors created by each block in `blocks`.
+    pub(crate) sapling_anchors_by_height: BTreeMap<block::Height, sapling::tree::Root>,
+    /// The Orchard anchors created by `blocks`.
+    pub(crate) orchard_anchors: MultiSet<orchard::tree::Root>,
+    /// The Orchard anchors created by each block in `blocks`.
+    pub(crate) orchard_anchors_by_height: BTreeMap<block::Height, orchard::tree::Root>,
+
+    /// The Sprout nullifiers revealed by `blocks`.
+    pub(super) sprout_nullifiers: HashSet<sprout::Nullifier>,
+    /// The Sapling nullifiers revealed by `blocks`.
+    pub(super) sapling_nullifiers: HashSet<sapling::Nullifier>,
+    /// The Orchard nullifiers revealed by `blocks`.
+    pub(super) orchard_nullifiers: HashSet<orchard::Nullifier>,
+
+    /// Partial transparent address index data from `blocks`.
+    pub(super) partial_transparent_transfers: HashMap<transparent::Address, TransparentTransfers>,
+
+    /// The cumulative work represented by `blocks`.
+    ///
+    /// Since the best chain is determined by the largest cumulative work,
+    /// the work represented by finalized blocks can be ignored,
+    /// because they are common to all non-finalized chains.
+    pub(super) partial_cumulative_work: PartialCumulativeWork,
+
+    /// The chain value pool balances of the tip of this [`Chain`],
+    /// including the block value pool changes from all finalized blocks,
+    /// and the non-finalized blocks in this chain.
+    ///
+    /// When a new chain is created from the finalized tip,
+    /// it is initialized with the finalized tip chain value pool balances.
+    pub(crate) chain_value_pools: ValueBalance<NonNegative>,
 }
 ```
 
@@ -293,7 +363,7 @@ Push a block into a chain as the new tip
     - Add the block's hash to `height_by_hash`
     - Add work to `self.partial_cumulative_work`
     - For each `transaction` in `block`
-      - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_by_hash`
+      - Add key: `transaction.hash` and value: `(height, tx_index)` to `tx_loc_by_hash`
       - Add created utxos to `self.created_utxos`
       - Add spent utxos to `self.spent_utxos`
       - Add nullifiers to the appropriate `self.<version>_nullifiers`
@@ -310,7 +380,7 @@ Remove the lowest height block of the non-finalized portion of a chain.
     - Remove the block's hash from `self.height_by_hash`
     - Subtract work from `self.partial_cumulative_work`
     - For each `transaction` in `block`
-      - Remove `transaction.hash` from `tx_by_hash`
+      - Remove `transaction.hash` from `tx_loc_by_hash`
       - Remove created utxos from `self.created_utxos`
       - Remove spent utxos from `self.spent_utxos`
       - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
@@ -340,7 +410,7 @@ Remove the highest height block of the non-finalized portion of a chain.
     - Remove the corresponding hash from `self.height_by_hash`
     - Subtract work from `self.partial_cumulative_work`
     - for each `transaction` in `block`
-      - remove `transaction.hash` from `tx_by_hash`
+      - remove `transaction.hash` from `tx_loc_by_hash`
       - Remove created utxos from `self.created_utxos`
       - Remove spent utxos from `self.spent_utxos`
       - Remove the nullifiers from the appropriate `self.<version>_nullifiers`
@@ -365,7 +435,7 @@ parent block is the tip of the finalized state. This implementation should be
 handled by `#[derive(Default)]`.
 
 1. initialise cumulative data members
-    - Construct an empty `self.blocks`, `height_by_hash`, `tx_by_hash`,
+    - Construct an empty `self.blocks`, `height_by_hash`, `tx_loc_by_hash`,
     `self.created_utxos`, `self.spent_utxos`, `self.<version>_anchors`,
     `self.<version>_nullifiers`
     - Zero `self.partial_cumulative_work`
@@ -1102,13 +1172,14 @@ Returns
 
 Implemented by querying:
 
-- (non-finalized) the `tx_by_hash` map (to get the block that contains the
+- (non-finalized) the `tx_loc_by_hash` map (to get the block that contains the
   transaction) of each chain starting with the best chain, and then find
   block that chain's `blocks` (to get the block containing the transaction
   data)
-- (finalized) the `tx_by_hash` tree (to get the block that contains the
-  transaction) and then `block_by_height` tree (to get the block containing
-  the transaction data), if the transaction is not in any non-finalized chain
+- (finalized) the `tx_loc_by_hash` tree (to get the block that contains the
+  transaction) and then `block_header_by_height` tree (to get the block
+  containing the transaction data), if the transaction is not in any
+  non-finalized chain
 
 ### `Request::Block(block::Hash)`
 [request-block]: #request-block
@@ -1125,8 +1196,9 @@ Implemented by querying:
 
 - (non-finalized) the `height_by_hash` of each chain starting with the best
   chain, then find block that chain's `blocks` (to get the block data)
-- (finalized) the `height_by_hash` tree (to get the block height) and then
-    the `block_by_height` tree (to get the block data), if the block is not in any non-finalized chain
+- (finalized) the `height_by_hash` tree (to get the block height) and then the
+  `block_header_by_height` tree (to get the block data), if the block is not in
+  any non-finalized chain
 
 ### `Request::AwaitSpendableUtxo { outpoint: OutPoint, spend_height: Height, spend_restriction: SpendRestriction }`
 

zebra-state/src/constants.rs

@@ -18,7 +18,7 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
 /// The database format version, incremented each time the database format changes.
-pub const DATABASE_FORMAT_VERSION: u32 = 24;
+pub const DATABASE_FORMAT_VERSION: u32 = 25;
 
 /// The maximum number of blocks to check for NU5 transactions,
 /// before we assume we are on a pre-NU5 legacy chain.

zebra-state/src/service/finalized_state/disk_db.rs

@@ -368,27 +368,24 @@ impl DiskDb {
 
         let column_families = vec![
             // Blocks
-            // TODO: rename to block_header_by_height (#3151)
-            rocksdb::ColumnFamilyDescriptor::new("block_by_height", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("hash_by_height", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("height_by_hash", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("block_header_by_height", db_options.clone()),
             // Transactions
             rocksdb::ColumnFamilyDescriptor::new("tx_by_loc", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new("hash_by_tx_loc", db_options.clone()),
-            // TODO: rename to tx_loc_by_hash (#3950)
-            rocksdb::ColumnFamilyDescriptor::new("tx_by_hash", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("tx_loc_by_hash", db_options.clone()),
             // Transparent
             rocksdb::ColumnFamilyDescriptor::new("balance_by_transparent_addr", db_options.clone()),
-            // TODO: #3951
-            //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()),
-            // TODO: rename to utxo_by_out_loc (#3952)
-            rocksdb::ColumnFamilyDescriptor::new("utxo_by_outpoint", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new(
-                "utxo_loc_by_transparent_addr_loc",
+                "tx_loc_by_transparent_addr_loc",
                 db_options.clone(),
             ),
+            // TODO: #3951
+            //rocksdb::ColumnFamilyDescriptor::new("tx_by_transparent_addr_loc", db_options.clone()),
+            rocksdb::ColumnFamilyDescriptor::new("utxo_by_out_loc", db_options.clone()),
             rocksdb::ColumnFamilyDescriptor::new(
-                "tx_loc_by_transparent_addr_loc",
+                "utxo_loc_by_transparent_addr_loc",
                 db_options.clone(),
             ),
             // Sprout

zebra-state/src/service/finalized_state/disk_format/tests/snapshots/block_header_by_height_raw_data@mainnet_0.snap

@@ -0,0 +1,11 @@
+---
+source: zebra-state/src/service/finalized_state/disk_format/tests/snapshot.rs
+assertion_line: 144
+expression: cf_data
+---
+[
+  KV(
+    k: "000000",
+    v: "040000000000000000000000000000000000000000000000000000000000000000000000db4d7a85b768123f1dff1d4c4cece70083b2d27e117b4ac2e31d087988a5eac4000000000000000000000000000000000000000000000000000000000000000090041358ffff071f5712000000000000000000000000000000000000000000000000000000000000fd4005000a889f00854b8665cd555f4656f68179d31ccadc1b1f7fb0952726313b16941da348284d67add4686121d4e3d930160c1348d8191c25f12b267a6a9c131b5031cbf8af1f79c9d513076a216ec87ed045fa966e01214ed83ca02dc1797270a454720d3206ac7d931a0a680c5c5e099057592570ca9bdf6058343958b31901fce1a15a4f38fd347750912e14004c73dfe588b903b6c03166582eeaf30529b14072a7b3079e3a684601b9b3024054201f7440b0ee9eb1a7120ff43f713735494aa27b1f8bab60d7f398bca14f6abb2adbf29b04099121438a7974b078a11635b594e9170f1086140b4173822dd697894483e1c6b4e8b8dcd5cb12ca4903bc61e108871d4d915a9093c18ac9b02b6716ce1013ca2c1174e319c1a570215bc9ab5f7564765f7be20524dc3fdf8aa356fd94d445e05ab165ad8bb4a0db096c097618c81098f91443c719416d39837af6de85015dca0de89462b1d8386758b2cf8a99e00953b308032ae44c35e05eb71842922eb69797f68813b59caf266cb6c213569ae3280505421a7e3a0a37fdf8e2ea354fc5422816655394a9454bac542a9298f176e211020d63dee6852c40de02267e2fc9d5e1ff2ad9309506f02a1a71a0501b16d0d36f70cdfd8de78116c0c506ee0b8ddfdeb561acadf31746b5a9dd32c21930884397fb1682164cb565cc14e089d66635a32618f7eb05fe05082b8a3fae620571660a6b89886eac53dec109d7cbb6930ca698a168f301a950be152da1be2b9e07516995e20baceebecb5579d7cdbc16d09f3a50cb3c7dffe33f26686d4ff3f8946ee6475e98cf7b3cf9062b6966e838f865ff3de5fb064a37a21da7bb8dfd2501a29e184f207caaba364f36f2329a77515dcb710e29ffbf73e2bbd773fab1f9a6b005567affff605c132e4e4dd69f36bd201005458cfbd2c658701eb2a700251cefd886b1e674ae816d3f719bac64be649c172ba27a4fd55947d95d53ba4cbc73de97b8af5ed4840b659370c556e7376457f51e5ebb66018849923db82c1c9a819f173cccdb8f3324b239609a300018d0fb094adf5bd7cbb3834c69e6d0b3798065c525b20f040e965e1a161af78ff7561cd874f5f1b75aa0bc77f720589e1b810f831eac5073e6dd46d00a2793f70f7427f0f798f2f53a67e615e65d356e66fe40609a958a05edb4c175bcc383ea0530e67ddbe479a898943c6e3074c6fcc252d6014de3a3d292b03f0d88d312fe221be7be7e3c59d07fa0f2f4029e364f1f355c5d01fa53770d0cd76d82bf7e60f6903bc1beb772e6fde4a70be51d9c7e03c8d6d8dfb361a234ba47c470fe630820bbd920715621b9fbedb49fcee165ead0875e6c2b1af16f50b5d6140cc981122fcbcf7c5a4e3772b3661b628e08380abc545957e59f634705b1bbde2f0b4e055a5ec5676d859be77e20962b645e051a880fddb0180b4555789e1f9344a436a84dc5579e2553f1e5fb0a599c137be36cabbed0319831fea3fddf94ddc7971e4bcf02cdc93294a9aab3e3b13e3b058235b4f4ec06ba4ceaa49d675b4ba80716f3bc6976b1fbf9c8bf1f3e3a4dc1cd83ef9cf816667fb94f1e923ff63fef072e6a19321e4812f96cb0ffa864da50ad74deb76917a336f31dce03ed5f0303aad5e6a83634f9fcc371096f8288b8f02ddded5ff1bb9d49331e4a84dbe1543164438fde9ad71dab024779dcdde0b6602b5ae0a6265c14b94edd83b37403f4b78fcd2ed555b596402c28ee81d87a909c4e8722b30c71ecdd861b05f61f8b1231795c76adba2fdefa451b283a5d527955b9f3de1b9828e7b2e74123dd47062ddcc09b05e7fa13cb2212a6fdbc65d7e852cec463ec6fd929f5b8483cf3052113b13dac91b69f49d1b7d1aec01c4a68e41ce157",
+  ),
+]