state: track UTXO provenance

This commit changes the state system and database format to track the
provenance of UTXOs, in addition to the outputs themselves.
Specifically, it tracks the following additional metadata:

- the height at which the UTXO was created;
- whether or not the UTXO was created from a coinbase transaction or
  not.

This metadata will allow us to:

- check the coinbase maturity consensus rule;
- check the coinbase inputs => no transparent outputs rule;
- implement lookup of transactions by utxo (using the height to find the
  block and then scanning the block) for a future RPC mechanism.

Closes #1342

zebra-consensus/src/block.rs

@@ -167,7 +167,7 @@ where
 
             let mut async_checks = FuturesUnordered::new();
 
-            let known_utxos = known_utxos(&block);
+            let known_utxos = new_outputs(&block);
             for transaction in &block.transactions {
                 let rsp = transaction_verifier
                     .ready_and()
@@ -221,15 +221,24 @@ where
     }
 }
 
-fn known_utxos(block: &Block) -> Arc<HashMap<transparent::OutPoint, transparent::Output>> {
-    let mut map = HashMap::default();
+fn new_outputs(block: &Block) -> Arc<HashMap<transparent::OutPoint, zs::Utxo>> {
+    let mut new_outputs = HashMap::default();
+    let height = block.coinbase_height().expect("block has coinbase height");
     for transaction in &block.transactions {
         let hash = transaction.hash();
+        let from_coinbase = transaction.is_coinbase();
         for (index, output) in transaction.outputs().iter().cloned().enumerate() {
             let index = index as u32;
-            map.insert(transparent::OutPoint { hash, index }, output);
+            new_outputs.insert(
+                transparent::OutPoint { hash, index },
+                zs::Utxo {
+                    output,
+                    height,
+                    from_coinbase,
+                },
+            );
         }
     }
 
-    Arc::new(map)
+    Arc::new(new_outputs)
 }

zebra-consensus/src/script.rs

@@ -3,6 +3,7 @@ use std::{collections::HashMap, future::Future, pin::Pin, sync::Arc};
 use tracing::Instrument;
 
 use zebra_chain::{parameters::ConsensusBranchId, transaction::Transaction, transparent};
+use zebra_state::Utxo;
 
 use crate::BoxError;
 
@@ -40,7 +41,7 @@ impl<ZS> Verifier<ZS> {
 pub struct Request {
     pub transaction: Arc<Transaction>,
     pub input_index: usize,
-    pub known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+    pub known_utxos: Arc<HashMap<transparent::OutPoint, Utxo>>,
 }
 
 impl<ZS> tower::Service<Request> for Verifier<ZS>
@@ -81,17 +82,21 @@ where
 
                 async move {
                     tracing::trace!("awaiting outpoint lookup");
-                    let output = if let Some(output) = known_utxos.get(&outpoint) {
+                    let utxo = if let Some(output) = known_utxos.get(&outpoint) {
                         tracing::trace!("UXTO in known_utxos, discarding query");
                         output.clone()
-                    } else if let zebra_state::Response::Utxo(output) = query.await? {
-                        output
+                    } else if let zebra_state::Response::Utxo(utxo) = query.await? {
+                        utxo
                     } else {
                         unreachable!("AwaitUtxo always responds with Utxo")
                     };
-                    tracing::trace!(?output, "got UTXO");
+                    tracing::trace!(?utxo, "got UTXO");
 
-                    zebra_script::is_valid(transaction, branch_id, (input_index as u32, output))?;
+                    zebra_script::is_valid(
+                        transaction,
+                        branch_id,
+                        (input_index as u32, utxo.output),
+                    )?;
                     tracing::trace!("script verification succeeded");
 
                     Ok(())

zebra-consensus/src/transaction.rs

@@ -56,13 +56,13 @@ pub enum Request {
     Block {
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
-        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+        known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
     },
     /// Verify the supplied transaction as part of the mempool.
     Mempool {
         transaction: Arc<Transaction>,
         /// Additional UTXOs which are known at the time of verification.
-        known_utxos: Arc<HashMap<transparent::OutPoint, transparent::Output>>,
+        known_utxos: Arc<HashMap<transparent::OutPoint, zs::Utxo>>,
     },
 }
 

zebra-state/Cargo.toml

@@ -36,4 +36,5 @@ spandoc = "0.2"
 tempdir = "0.3.7"
 tokio = { version = "0.3", features = ["full"] }
 proptest = "0.10.1"
+proptest-derive = "0.2"
 primitive-types = "0.7.3"

zebra-state/src/constants.rs

@@ -11,4 +11,5 @@ pub const MIN_TRASPARENT_COINBASE_MATURITY: u32 = 100;
 /// coinbase transactions.
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRASPARENT_COINBASE_MATURITY - 1;
 
-pub const DATABASE_FORMAT_VERSION: u32 = 2;
+/// The database format version, incremented each time the database format changes.
+pub const DATABASE_FORMAT_VERSION: u32 = 3;

zebra-state/src/lib.rs

@@ -15,6 +15,7 @@ mod request;
 mod response;
 mod service;
 mod util;
+mod utxo;
 
 // TODO: move these to integration tests.
 #[cfg(test)]
@@ -26,3 +27,4 @@ pub use error::{BoxError, CloneError, CommitBlockError, ValidateContextError};
 pub use request::{FinalizedBlock, HashOrHeight, PreparedBlock, Request};
 pub use response::Response;
 pub use service::init;
+pub use utxo::Utxo;

zebra-state/src/request.rs

@@ -5,6 +5,8 @@ use zebra_chain::{
     transaction, transparent,
 };
 
+use crate::Utxo;
+
 // Allow *only* this unused import, so that rustdoc link resolution
 // will work with inline links.
 #[allow(unused_imports)]
@@ -71,7 +73,7 @@ pub struct PreparedBlock {
     /// Note: although these transparent outputs are newly created, they may not
     /// be unspent, since a later transaction in a block can spend outputs of an
     /// earlier transaction.
-    pub new_outputs: HashMap<transparent::OutPoint, transparent::Output>,
+    pub new_outputs: HashMap<transparent::OutPoint, Utxo>,
     // TODO: add these parameters when we can compute anchors.
     // sprout_anchor: sprout::tree::Root,
     // sapling_anchor: sapling::tree::Root,
@@ -88,6 +90,13 @@ pub struct FinalizedBlock {
     pub(crate) block: Arc<Block>,
     pub(crate) hash: block::Hash,
     pub(crate) height: block::Height,
+    /// New transparent outputs created in this block, indexed by
+    /// [`Outpoint`](transparent::Outpoint).
+    ///
+    /// Note: although these transparent outputs are newly created, they may not
+    /// be unspent, since a later transaction in a block can spend outputs of an
+    /// earlier transaction.
+    pub(crate) new_outputs: HashMap<transparent::OutPoint, Utxo>,
 }
 
 // Doing precomputation in this From impl means that it will be done in
@@ -100,10 +109,45 @@ impl From<Arc<Block>> for FinalizedBlock {
             .expect("finalized blocks must have a valid coinbase height");
         let hash = block.hash();
 
+        let mut new_outputs = HashMap::default();
+        for transaction in &block.transactions {
+            let hash = transaction.hash();
+            let from_coinbase = transaction.is_coinbase();
+            for (index, output) in transaction.outputs().iter().cloned().enumerate() {
+                let index = index as u32;
+                new_outputs.insert(
+                    transparent::OutPoint { hash, index },
+                    Utxo {
+                        output,
+                        height,
+                        from_coinbase,
+                    },
+                );
+            }
+        }
+
+        Self {
+            block,
+            height,
+            hash,
+            new_outputs,
+        }
+    }
+}
+
+impl From<PreparedBlock> for FinalizedBlock {
+    fn from(prepared: PreparedBlock) -> Self {
+        let PreparedBlock {
+            block,
+            height,
+            hash,
+            new_outputs,
+        } = prepared;
         Self {
             block,
             height,
             hash,
+            new_outputs,
         }
     }
 }

zebra-state/src/response.rs

@@ -2,9 +2,10 @@ use std::sync::Arc;
 use zebra_chain::{
     block::{self, Block},
     transaction::Transaction,
-    transparent,
 };
 
+use crate::Utxo;
+
 // Allow *only* this unused import, so that rustdoc link resolution
 // will work with inline links.
 #[allow(unused_imports)]
@@ -33,5 +34,5 @@ pub enum Response {
     Block(Option<Arc<Block>>),
 
     /// The response to a `AwaitUtxo` request
-    Utxo(transparent::Output),
+    Utxo(Utxo),
 }

zebra-state/src/service.rs

@@ -21,7 +21,7 @@ use zebra_chain::{
 
 use crate::{
     request::HashOrHeight, BoxError, CommitBlockError, Config, FinalizedBlock, PreparedBlock,
-    Request, Response, ValidateContextError,
+    Request, Response, Utxo, ValidateContextError,
 };
 
 use self::finalized_state::FinalizedState;
@@ -251,12 +251,12 @@ impl StateService {
             .or_else(|| self.disk.height(hash))
     }
 
-    /// Return the utxo pointed to by `outpoint` if it exists in any chain.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    /// Return the [`Utxo`] pointed to by `outpoint` if it exists in any chain.
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         self.mem
             .utxo(outpoint)
-            .or_else(|| self.disk.utxo(outpoint))
             .or_else(|| self.queued_blocks.utxo(outpoint))
+            .or_else(|| self.disk.utxo(outpoint))
     }
 
     /// Return an iterator over the relevant chain of the block identified by
@@ -404,7 +404,7 @@ impl Service<Request> for StateService {
 
                 let (rsp_tx, rsp_rx) = oneshot::channel();
 
-                self.pending_utxos.scan_block(&finalized.block);
+                self.pending_utxos.check_against(&finalized.new_outputs);
                 self.disk.queue_and_commit_finalized((finalized, rsp_tx));
 
                 async move {

zebra-state/src/service/finalized_state.rs

@@ -11,7 +11,7 @@ use zebra_chain::{
     transaction::{self, Transaction},
 };
 
-use crate::{BoxError, Config, FinalizedBlock, HashOrHeight};
+use crate::{BoxError, Config, FinalizedBlock, HashOrHeight, Utxo};
 
 use self::disk_format::{DiskDeserialize, DiskSerialize, FromDisk, IntoDisk, TransactionLocation};
 
@@ -149,6 +149,7 @@ impl FinalizedState {
             block,
             hash,
             height,
+            new_outputs,
         } = finalized;
 
         let hash_by_height = self.db.cf_handle("hash_by_height").unwrap();
@@ -205,7 +206,13 @@ impl FinalizedState {
                 return batch;
             }
 
-            // Index each transaction
+            // Index all new transparent outputs
+            for (outpoint, utxo) in new_outputs.into_iter() {
+                batch.zs_insert(utxo_by_outpoint, outpoint, utxo);
+            }
+
+            // Index each transaction, spent inputs, nullifiers
+            // TODO: move computation into FinalizedBlock as with transparent outputs
             for (transaction_index, transaction) in block.transactions.iter().enumerate() {
                 let transaction_hash = transaction.hash();
                 let transaction_location = TransactionLocation {
@@ -228,15 +235,6 @@ impl FinalizedState {
                     }
                 }
 
-                // Index all new transparent outputs
-                for (index, output) in transaction.outputs().iter().enumerate() {
-                    let outpoint = transparent::OutPoint {
-                        hash: transaction_hash,
-                        index: index as _,
-                    };
-                    batch.zs_insert(utxo_by_outpoint, outpoint, output);
-                }
-
                 // Mark sprout and sapling nullifiers as spent
                 for sprout_nullifier in transaction.sprout_nullifiers() {
                     batch.zs_insert(sprout_nullifiers, sprout_nullifier, ());
@@ -305,7 +303,7 @@ impl FinalizedState {
 
     /// Returns the `transparent::Output` pointed to by the given
     /// `transparent::OutPoint` if it is present.
-    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<transparent::Output> {
+    pub fn utxo(&self, outpoint: &transparent::OutPoint) -> Option<Utxo> {
         let utxo_by_outpoint = self.db.cf_handle("utxo_by_outpoint").unwrap();
         self.db.zs_get(utxo_by_outpoint, outpoint)
     }

zebra-state/src/service/finalized_state/disk_format.rs

@@ -5,10 +5,12 @@ use zebra_chain::{
     block,
     block::Block,
     sapling,
-    serialization::{ZcashDeserialize, ZcashSerialize},
+    serialization::{ZcashDeserialize, ZcashDeserializeInto, ZcashSerialize},
     sprout, transaction, transparent,
 };
 
+use crate::Utxo;
+
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub struct TransactionLocation {
     pub height: block::Height,
@@ -184,19 +186,33 @@ impl FromDisk for block::Height {
     }
 }
 
-impl IntoDisk for transparent::Output {
+impl IntoDisk for Utxo {
     type Bytes = Vec<u8>;
 
     fn as_bytes(&self) -> Self::Bytes {
-        self.zcash_serialize_to_vec()
-            .expect("serialization to vec doesn't fail")
+        let mut bytes = vec![0; 5];
+        bytes[0..4].copy_from_slice(&self.height.0.to_be_bytes());
+        bytes[4] = self.from_coinbase as u8;
+        self.output
+            .zcash_serialize(&mut bytes)
+            .expect("serialization to vec doesn't fail");
+        bytes
     }
 }
 
-impl FromDisk for transparent::Output {
+impl FromDisk for Utxo {
     fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
-        Self::zcash_deserialize(bytes.as_ref())
-            .expect("deserialization format should match the serialization format used by IntoDisk")
+        let (meta_bytes, output_bytes) = bytes.as_ref().split_at(5);
+        let height = block::Height(u32::from_be_bytes(meta_bytes[0..4].try_into().unwrap()));
+        let from_coinbase = meta_bytes[4] == 1u8;
+        let output = output_bytes
+            .zcash_deserialize_into()
+            .expect("db has serialized data");
+        Self {
+            output,
+            height,
+            from_coinbase,
+        }
     }
 }
 
@@ -370,6 +386,6 @@ mod tests {
     fn roundtrip_transparent_output() {
         zebra_test::init();
 
-        proptest!(|(val in any::<transparent::Output>())| assert_value_properties(val));
+        proptest!(|(val in any::<Utxo>())| assert_value_properties(val));
     }
 }