feat!: add recovery byte to output features (#3727)

Description
---
Added `recovery_byte` to `OutputFeatures`. The recovery byte will uniformly distribute all outputs in the blockchain into 255 "bins" that can be identified by the commitment of the output in combination with a master recovery byte key.

This change will come into effect on `dibbler` testnet once the `OutputFeatures` version is updated from `V0` to `V1`, which will result in a hard fork.

All funds will be retained after the hard fork.

Motivation and Context
---
The reasoning behind this is that it will enable a client to request only a specific subset of full-on outputs in a set of blocks. The commitment only takes up a fraction (less than 5%) of the byte space for a full-on output. A typical use case for this feature would be a wallet recovery client that needs to scan all outputs in the blockchain from the birthdate of the wallet to be recovered. The client would request all commitments in the set of blocks from a base node and identify all those that belong to their own "bin", after which a tailor made request can be submitted to retrieve the full-on output data for the identified commitments. This will result in a better than 20x saving in the amount of data transmitted for wallet recovery.

How Has This Been Tested?
---
- Unit tests
- Cucumber integration tests
- Performed system-level tests using a wallet that contains embedded output features in json text that does not contain the recovery byte.

applications/tari_app_grpc/proto/types.proto

@@ -226,6 +226,10 @@ message OutputFeatures {
     // Version
     uint32 version = 9;
     CommitteeDefinitionFeatures committee_definition = 10;
+
+    // The recovery byte - not consensus critical - can help reduce the bandwidth with wallet recovery or in other
+    // instances when a wallet needs to request the complete UTXO set from a base node.
+    uint32 recovery_byte = 11;
 }
 
 message AssetOutputFeatures {

applications/tari_app_grpc/src/conversions/output_features.rs

@@ -62,6 +62,7 @@ impl TryFrom<grpc::OutputFeatures> for OutputFeatures {
             OutputFlags::from_bits(features.flags as u8)
                 .ok_or_else(|| "Invalid or unrecognised output flags".to_string())?,
             features.maturity,
+            u8::try_from(features.recovery_byte).map_err(|_| "Invalid recovery byte: overflowed u8")?,
             features.metadata,
             unique_id,
             parent_public_key,
@@ -89,6 +90,7 @@ impl From<OutputFeatures> for grpc::OutputFeatures {
             sidechain_checkpoint: features.sidechain_checkpoint.map(|m| m.into()),
             version: features.version as u32,
             committee_definition: features.committee_definition.map(|c| c.into()),
+            recovery_byte: features.recovery_byte as u32,
         }
     }
 }

applications/test_faucet/src/main.rs

@@ -147,7 +147,7 @@ impl Iterator for UTXOFeatures {
     type Item = OutputFeatures;
 
     fn next(&mut self) -> Option<Self::Item> {
-        let f = OutputFeatures::with_maturity(0);
+        let f = OutputFeatures::default();
         Some(f)
     }
 }

base_layer/core/src/blocks/genesis_block.rs

@@ -243,6 +243,7 @@ fn get_dibbler_genesis_block_raw() -> Block {
                 version:OutputFeaturesVersion::V0,
                 flags:OutputFlags::COINBASE_OUTPUT,
                 maturity:60,
+                recovery_byte: 0,
                 metadata: Vec::new(),
                 unique_id: None,
                 parent_public_key: None,

base_layer/core/src/chain_storage/tests/blockchain_database.rs

@@ -388,7 +388,10 @@ mod add_block {
 
         let prev_block = blocks.last().unwrap();
         // Used to help identify the output we're interrogating in this test
-        let features = OutputFeatures::with_maturity(1);
+        let features = OutputFeatures {
+            maturity: 1,
+            ..Default::default()
+        };
         let (txns, tx_outputs) = schema_to_transaction(&[txn_schema!(
             from: vec![outputs[0].clone()],
             to: vec![500 * T],
@@ -731,7 +734,7 @@ mod fetch_utxo_by_unique_id {
         // Height 1
         let (blocks, outputs) = add_many_chained_blocks(1, &db);
 
-        let features = OutputFeatures {
+        let mut features = OutputFeatures {
             flags: OutputFlags::MINT_NON_FUNGIBLE,
             parent_public_key: Some(asset_pk.clone()),
             unique_id: Some(unique_id.clone()),
@@ -742,8 +745,9 @@ mod fetch_utxo_by_unique_id {
             to: vec![500 * T],
             fee: 5.into(),
             lock: 0,
-            features: features
+            features: features.clone()
         )]);
+        features.set_recovery_byte(tx_outputs[0].features.recovery_byte);
 
         let asset_utxo1 = tx_outputs.iter().find(|o| o.features == features).unwrap();
 
@@ -766,7 +770,7 @@ mod fetch_utxo_by_unique_id {
             expected_commitment
         );
 
-        let features = OutputFeatures {
+        let mut features = OutputFeatures {
             flags: OutputFlags::empty(),
             parent_public_key: Some(asset_pk.clone()),
             unique_id: Some(unique_id.clone()),
@@ -779,6 +783,7 @@ mod fetch_utxo_by_unique_id {
             lock: 0,
             features: features
         )]);
+        features.set_recovery_byte(tx_outputs[0].features.recovery_byte);
 
         let asset_utxo2 = tx_outputs.iter().find(|o| o.features == features).unwrap();
 

base_layer/core/src/consensus/consensus_constants.rs

@@ -180,8 +180,9 @@ impl ConsensusConstants {
 
     pub fn coinbase_weight(&self) -> u64 {
         // TODO: We do not know what script, features etc a coinbase has - this should be max coinbase size?
+        let output_features = OutputFeatures { ..Default::default() };
         let metadata_size = self.transaction_weight.round_up_metadata_size(
-            script![Nop].consensus_encode_exact_size() + OutputFeatures::default().consensus_encode_exact_size(),
+            script![Nop].consensus_encode_exact_size() + output_features.consensus_encode_exact_size(),
         );
         self.transaction_weight.calculate(1, 0, 1, metadata_size)
     }

base_layer/core/src/covenants/covenant.rs

@@ -181,7 +181,13 @@ mod test {
         outputs[7].features.maturity = 42;
         let mut input = create_input();
         input.set_maturity(42).unwrap();
-        let covenant = covenant!(fields_preserved(@fields(@field::features)));
+        let covenant = covenant!(fields_preserved(@fields(
+            @field::features_flags,
+            @field::features_maturity,
+            @field::features_unique_id,
+            @field::features_parent_public_key,
+            @field::features_metadata))
+        );
         let num_matching_outputs = covenant.execute(0, &input, &outputs).unwrap();
         assert_eq!(num_matching_outputs, 3);
     }

base_layer/core/src/covenants/fields.rs

@@ -335,7 +335,7 @@ mod test {
 
     #[test]
     fn get_field_value_ref() {
-        let features = OutputFeatures {
+        let mut features = OutputFeatures {
             maturity: 42,
             ..Default::default()
         };
@@ -345,6 +345,7 @@ mod test {
         })
         .pop()
         .unwrap();
+        features.set_recovery_byte(output.features.recovery_byte);
         let r = OutputField::Features.get_field_value_ref::<OutputFeatures>(&output);
         assert_eq!(*r.unwrap(), features);
     }

base_layer/core/src/proto/transaction.proto

@@ -96,6 +96,10 @@ message OutputFeatures {
     // Version
     uint32 version = 9;
     CommitteeDefinitionFeatures committee_definition = 10;
+
+    // The recovery byte - not consensus critical - can help reduce the bandwidth with wallet recovery or in other
+    // instances when a wallet needs to request the complete UTXO set from a base node.
+    uint32 recovery_byte = 11;
 }
 
 message AssetOutputFeatures {

base_layer/core/src/proto/transaction.rs

@@ -290,6 +290,7 @@ impl TryFrom<proto::types::OutputFeatures> for OutputFeatures {
             OutputFlags::from_bits(features.flags as u8)
                 .ok_or_else(|| "Invalid or unrecognised output flags".to_string())?,
             features.maturity,
+            u8::try_from(features.recovery_byte).map_err(|_| "Invalid recovery byte: overflowed u8")?,
             features.metadata,
             unique_id,
             parent_public_key,
@@ -323,6 +324,7 @@ impl From<OutputFeatures> for proto::types::OutputFeatures {
             sidechain_checkpoint: features.sidechain_checkpoint.map(|s| s.into()),
             version: features.version as u32,
             committee_definition: features.committee_definition.map(|c| c.into()),
+            recovery_byte: features.recovery_byte as u32,
         }
     }
 }

base_layer/core/src/test_helpers/mod.rs

@@ -110,13 +110,13 @@ pub fn mine_to_difficulty(mut block: Block, difficulty: Difficulty) -> Result<Bl
     // When starting from the same nonce, in tests it becomes common to mine the same block more than once without the
     // hash changing. This introduces the required entropy
     block.header.nonce = rand::thread_rng().gen();
-    for _i in 0..10000 {
+    for _i in 0..20000 {
         if sha3_difficulty(&block.header) == difficulty {
             return Ok(block);
         }
         block.header.nonce += 1;
     }
-    Err("Could not mine to difficulty in 10000 iterations".to_string())
+    Err("Could not mine to difficulty in 20000 iterations".to_string())
 }
 
 pub fn create_peer_manager<P: AsRef<Path>>(data_path: P) -> Arc<PeerManager> {

base_layer/core/src/transactions/coinbase_builder.rs

@@ -188,7 +188,13 @@ impl CoinbaseBuilder {
         let spending_key = self.spend_key.ok_or(CoinbaseBuildError::MissingSpendKey)?;
         let script_private_key = self.script_key.unwrap_or_else(|| spending_key.clone());
         let script = self.script.unwrap_or_else(|| script!(Nop));
-        let output_features = OutputFeatures::create_coinbase(height + constants.coinbase_lock_height());
+
+        let commitment = self
+            .factories
+            .commitment
+            .commit_value(&spending_key, total_reward.as_u64());
+        let recovery_byte = OutputFeatures::create_unique_recovery_byte(&commitment, self.rewind_data.as_ref());
+        let output_features = OutputFeatures::create_coinbase(height + constants.coinbase_lock_height(), recovery_byte);
         let excess = self.factories.commitment.commit_value(&spending_key, 0);
         let kernel_features = KernelFeatures::create_coinbase();
         let metadata = TransactionMetadata::default();
@@ -286,7 +292,7 @@ mod test {
         let (builder, rules, _) = get_builder();
         assert_eq!(
             builder
-                .build(rules.consensus_constants(0), rules.emission_schedule())
+                .build(rules.consensus_constants(0), rules.emission_schedule(),)
                 .unwrap_err(),
             CoinbaseBuildError::MissingBlockHeight
         );
@@ -298,7 +304,7 @@ mod test {
         let builder = builder.with_block_height(42);
         assert_eq!(
             builder
-                .build(rules.consensus_constants(42), rules.emission_schedule())
+                .build(rules.consensus_constants(42), rules.emission_schedule(),)
                 .unwrap_err(),
             CoinbaseBuildError::MissingFees
         );
@@ -313,7 +319,7 @@ mod test {
         let builder = builder.with_block_height(42).with_fees(fees).with_nonce(p.nonce);
         assert_eq!(
             builder
-                .build(rules.consensus_constants(42), rules.emission_schedule())
+                .build(rules.consensus_constants(42), rules.emission_schedule(),)
                 .unwrap_err(),
             CoinbaseBuildError::MissingSpendKey
         );
@@ -358,6 +364,7 @@ mod test {
         let rewind_data = RewindData {
             rewind_key: rewind_key.clone(),
             rewind_blinding_key: rewind_blinding_key.clone(),
+            recovery_byte_key: PrivateKey::random(&mut OsRng),
             proof_message: proof_message.to_owned(),
         };