feat: use CipherSeed wallet birthday for recovery start point (#3602)

Description
---
This PR makes use of the wallet birthday encoded into the wallet’s CipherSeed as a starting point for wallet recovery. This is instead of starting at the genesis block and will reduce the amount of work the wallet needs to do significantly by exclude all the blocks before its birthday.

This PR implements a new RPC method on the BaseNodeSyncRpcService service called `get_height_at_time` which accepts a Unix Epoch time. The base node will then use a binary search strategy to determine what the block height was at that time. When a fresh Wallet Recovery is started if there isn’t a current progress metadata already stored in the database the wallet will calculate the unix epoch time of two days prior CipherSeeds birthday. This is to account for any timezone issues and does not add much in terms of work to the process. The wallet will then request the height at this time, request the header for that height and will be able to start the recovery process from that point.

How Has This Been Tested?
---
Tests provided for RPC service and CipherSeed birthday db storage.
UTXO Recovery tested manually

base_layer/core/src/base_node/sync/rpc/mod.rs

@@ -90,6 +90,9 @@ pub trait BaseNodeSyncService: Send + Sync + 'static {
 
     #[rpc(method = 8)]
     async fn sync_utxos(&self, request: Request<SyncUtxosRequest>) -> Result<Streaming<SyncUtxosResponse>, RpcStatus>;
+
+    #[rpc(method = 9)]
+    async fn get_height_at_time(&self, request: Request<u64>) -> Result<Response<u64>, RpcStatus>;
 }
 
 #[cfg(feature = "base_node")]

base_layer/core/src/base_node/sync/rpc/service.rs

@@ -462,4 +462,59 @@ impl<B: BlockchainBackend + 'static> BaseNodeSyncService for BaseNodeSyncRpcServ
 
         Ok(Streaming::new(rx))
     }
+
+    async fn get_height_at_time(&self, request: Request<u64>) -> Result<Response<u64>, RpcStatus> {
+        let requested_epoch_time: u64 = request.into_message();
+
+        let tip_header = self
+            .db()
+            .fetch_tip_header()
+            .await
+            .map_err(RpcStatus::log_internal_error(LOG_TARGET))?;
+        let mut left_height = 0u64;
+        let mut right_height = tip_header.height();
+
+        while left_height <= right_height {
+            let mut mid_height = (left_height + right_height) / 2;
+
+            if mid_height == 0 {
+                return Ok(Response::new(0u64));
+            }
+            // If the two bounds are adjacent then perform the test between the right and left sides
+            if left_height == mid_height {
+                mid_height = right_height;
+            }
+
+            let mid_header = self
+                .db()
+                .fetch_header(mid_height)
+                .await
+                .map_err(RpcStatus::log_internal_error(LOG_TARGET))?
+                .ok_or_else(|| {
+                    RpcStatus::not_found(format!("Header not found during search at height {}", mid_height))
+                })?;
+            let before_mid_header = self
+                .db()
+                .fetch_header(mid_height - 1)
+                .await
+                .map_err(RpcStatus::log_internal_error(LOG_TARGET))?
+                .ok_or_else(|| {
+                    RpcStatus::not_found(format!("Header not found during search at height {}", mid_height - 1))
+                })?;
+
+            if requested_epoch_time < mid_header.timestamp.as_u64() &&
+                requested_epoch_time >= before_mid_header.timestamp.as_u64()
+            {
+                return Ok(Response::new(before_mid_header.height));
+            } else if mid_height == right_height {
+                return Ok(Response::new(right_height));
+            } else if requested_epoch_time <= mid_header.timestamp.as_u64() {
+                right_height = mid_height;
+            } else {
+                left_height = mid_height;
+            }
+        }
+
+        Ok(Response::new(0u64))
+    }
 }

base_layer/core/tests/base_node_rpc.rs

@@ -42,7 +42,7 @@
 //  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-use std::convert::TryFrom;
+use std::{convert::TryFrom, sync::Arc, time::Duration};
 
 use randomx_rs::RandomXFlag;
 use tempfile::{tempdir, TempDir};
@@ -61,6 +61,8 @@ use tari_core::{
         },
         rpc::{BaseNodeWalletRpcService, BaseNodeWalletService},
         state_machine_service::states::{ListeningInfo, StateInfo, StatusInfo},
+        sync::rpc::BaseNodeSyncRpcService,
+        BaseNodeSyncService,
     },
     blocks::ChainBlock,
     consensus::{ConsensusManager, ConsensusManagerBuilder, NetworkConsensus},
@@ -80,14 +82,15 @@ use tari_core::{
 };
 
 use crate::helpers::{
-    block_builders::{chain_block, create_genesis_block_with_coinbase_value},
+    block_builders::{chain_block, chain_block_with_new_coinbase, create_genesis_block_with_coinbase_value},
     nodes::{BaseNodeBuilder, NodeInterfaces},
 };
 
 mod helpers;
 
 async fn setup() -> (
     BaseNodeWalletRpcService<TempDatabase>,
+    BaseNodeSyncRpcService<TempDatabase>,
     NodeInterfaces,
     RpcRequestMock,
     ConsensusManager,
@@ -118,13 +121,15 @@ async fn setup() -> (
     });
 
     let request_mock = RpcRequestMock::new(base_node.comms.peer_manager());
-    let service = BaseNodeWalletRpcService::new(
+    let wallet_service = BaseNodeWalletRpcService::new(
         base_node.blockchain_db.clone().into(),
         base_node.mempool_handle.clone(),
         base_node.state_machine_handle.clone(),
     );
+    let base_node_service = BaseNodeSyncRpcService::new(base_node.blockchain_db.clone().into());
     (
-        service,
+        wallet_service,
+        base_node_service,
         base_node,
         request_mock,
         consensus_manager,
@@ -138,7 +143,7 @@ async fn setup() -> (
 #[allow(clippy::identity_op)]
 async fn test_base_node_wallet_rpc() {
     // Testing the submit_transaction() and transaction_query() rpc calls
-    let (service, mut base_node, request_mock, consensus_manager, block0, utxo0, _temp_dir) = setup().await;
+    let (service, _, mut base_node, request_mock, consensus_manager, block0, utxo0, _temp_dir) = setup().await;
 
     let (txs1, utxos1) = schema_to_transaction(&[txn_schema!(from: vec![utxo0.clone()], to: vec![1 * T, 1 * T])]);
     let tx1 = (*txs1[0]).clone();
@@ -290,3 +295,64 @@ async fn test_base_node_wallet_rpc() {
             .any(|u| u.as_transaction_output(&factories).unwrap().commitment == output.commitment));
     }
 }
+
+#[tokio::test]
+async fn test_get_height_at_time() {
+    let factories = CryptoFactories::default();
+
+    let (_, service, base_node, request_mock, consensus_manager, block0, _utxo0, _temp_dir) = setup().await;
+
+    let mut prev_block = block0.clone();
+    let mut times = Vec::new();
+    times.push(prev_block.header().timestamp);
+    for _ in 0..10 {
+        tokio::time::sleep(Duration::from_secs(2)).await;
+        let new_block = base_node
+            .blockchain_db
+            .prepare_new_block(chain_block_with_new_coinbase(&prev_block, vec![], &consensus_manager, &factories).0)
+            .unwrap();
+
+        prev_block = base_node
+            .blockchain_db
+            .add_block(Arc::new(new_block))
+            .unwrap()
+            .assert_added();
+        times.push(prev_block.header().timestamp);
+    }
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64() - 100);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 7);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64() - 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 6);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 7);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 10);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64() - 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 9);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 10);
+}

base_layer/key_manager/src/cipher_seed.rs

@@ -85,7 +85,7 @@ pub const CIPHER_SEED_MAC_BYTES: usize = 5;
 pub struct CipherSeed {
     version: u8,
     birthday: u16,
-    pub entropy: [u8; CIPHER_SEED_ENTROPY_BYTES],
+    entropy: [u8; CIPHER_SEED_ENTROPY_BYTES],
     salt: [u8; CIPHER_SEED_SALT_BYTES],
 }
 
@@ -108,7 +108,7 @@ impl CipherSeed {
 
     pub fn encipher(&self, passphrase: Option<String>) -> Result<Vec<u8>, KeyManagerError> {
         let mut plaintext = self.birthday.to_le_bytes().to_vec();
-        plaintext.append(&mut self.entropy.clone().to_vec());
+        plaintext.append(&mut self.entropy().clone().to_vec());
 
         let passphrase = passphrase.unwrap_or_else(|| DEFAULT_CIPHER_SEED_PASSPHRASE.to_string());
 
@@ -236,6 +236,14 @@ impl CipherSeed {
 
         Ok(())
     }
+
+    pub fn entropy(&self) -> [u8; CIPHER_SEED_ENTROPY_BYTES] {
+        self.entropy
+    }
+
+    pub fn birthday(&self) -> u16 {
+        self.birthday
+    }
 }
 
 impl Drop for CipherSeed {

base_layer/key_manager/src/key_manager.rs

@@ -74,7 +74,7 @@ where
 
     /// Derive a new private key from master key: derived_key=SHA256(master_key||branch_seed||index)
     pub fn derive_key(&self, key_index: u64) -> Result<DerivedKey<K>, ByteArrayError> {
-        let concatenated = format!("{}{}", self.seed.entropy.to_vec().to_hex(), key_index.to_string());
+        let concatenated = format!("{}{}", self.seed.entropy().to_vec().to_hex(), key_index.to_string());
         match K::from_bytes(D::digest(&concatenated.into_bytes()).as_slice()) {
             Ok(k) => Ok(DerivedKey { k, key_index }),
             Err(e) => Err(e),

base_layer/wallet/src/storage/database.rs

@@ -55,6 +55,7 @@ pub enum DbKey {
     MasterSeed,
     PassphraseHash,
     EncryptionSalt,
+    WalletBirthday,
 }
 
 pub enum DbValue {
@@ -67,6 +68,7 @@ pub enum DbValue {
     MasterSeed(CipherSeed),
     PassphraseHash(String),
     EncryptionSalt(String),
+    WalletBirthday(String),
 }
 
 #[derive(Clone)]
@@ -306,6 +308,22 @@ where T: WalletBackend + 'static
         .map_err(|err| WalletStorageError::BlockingTaskSpawnError(err.to_string()))??;
         Ok(c)
     }
+
+    pub async fn get_wallet_birthday(&self) -> Result<u16, WalletStorageError> {
+        let db_clone = self.db.clone();
+
+        let result = tokio::task::spawn_blocking(move || match db_clone.fetch(&DbKey::WalletBirthday) {
+            Ok(None) => Err(WalletStorageError::ValueNotFound(DbKey::WalletBirthday)),
+            Ok(Some(DbValue::WalletBirthday(b))) => Ok(b
+                .parse::<u16>()
+                .map_err(|_| WalletStorageError::ConversionError("Could not parse wallet birthday".to_string()))?),
+            Ok(Some(other)) => unexpected_result(DbKey::WalletBirthday, other),
+            Err(e) => log_error(DbKey::WalletBirthday, e),
+        })
+        .await
+        .map_err(|err| WalletStorageError::BlockingTaskSpawnError(err.to_string()))??;
+        Ok(result)
+    }
 }
 
 impl Display for DbKey {
@@ -319,6 +337,7 @@ impl Display for DbKey {
             DbKey::BaseNodeChainMetadata => f.write_str(&"Last seen Chain metadata from base node".to_string()),
             DbKey::PassphraseHash => f.write_str(&"PassphraseHash".to_string()),
             DbKey::EncryptionSalt => f.write_str(&"EncryptionSalt".to_string()),
+            DbKey::WalletBirthday => f.write_str(&"WalletBirthday".to_string()),
         }
     }
 }
@@ -335,6 +354,7 @@ impl Display for DbValue {
             DbValue::BaseNodeChainMetadata(v) => f.write_str(&format!("Last seen Chain metadata from base node:{}", v)),
             DbValue::PassphraseHash(h) => f.write_str(&format!("PassphraseHash: {}", h)),
             DbValue::EncryptionSalt(s) => f.write_str(&format!("EncryptionSalt: {}", s)),
+            DbValue::WalletBirthday(b) => f.write_str(&format!("WalletBirthday: {}", b)),
         }
     }
 }

base_layer/wallet/src/storage/sqlite_db.rs

@@ -83,7 +83,9 @@ impl WalletSqliteDatabase {
         match cipher.as_ref() {
             None => {
                 let seed_bytes = seed.encipher(None)?;
+                let birthday = seed.birthday();
                 WalletSettingSql::new(DbKey::MasterSeed.to_string(), seed_bytes.to_hex()).set(conn)?;
+                WalletSettingSql::new(DbKey::WalletBirthday.to_string(), birthday.to_string()).set(conn)?;
             },
             Some(cipher) => {
                 let seed_bytes = seed.encipher(None)?;
@@ -305,6 +307,9 @@ impl WalletSqliteDatabase {
             DbKey::EncryptionSalt => {
                 return Err(WalletStorageError::OperationNotSupported);
             },
+            DbKey::WalletBirthday => {
+                return Err(WalletStorageError::OperationNotSupported);
+            },
         };
         if start.elapsed().as_millis() > 0 {
             trace!(
@@ -346,6 +351,7 @@ impl WalletBackend for WalletSqliteDatabase {
             DbKey::BaseNodeChainMetadata => self.get_chain_metadata(&conn)?.map(DbValue::BaseNodeChainMetadata),
             DbKey::PassphraseHash => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::PassphraseHash),
             DbKey::EncryptionSalt => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::EncryptionSalt),
+            DbKey::WalletBirthday => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::WalletBirthday),
         };
         if start.elapsed().as_millis() > 0 {
             trace!(

base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs

@@ -338,14 +338,15 @@ where TBackend: WalletBackend + 'static
     }
 
     async fn get_start_utxo_mmr_pos(&self, client: &mut BaseNodeSyncRpcClient) -> Result<u64, UtxoScannerError> {
-        let metadata = self.get_metadata().await?.unwrap_or_default();
-        if metadata.height_hash.is_empty() {
-            // Set a value in here so that if the recovery fails on the genesis block the client will know a
-            // recover was started. Important on Console wallet that otherwise makes this decision based on the
-            // presence of the data file
-            self.set_metadata(metadata).await?;
-            return Ok(0);
-        }
+        let metadata = match self.get_metadata().await? {
+            None => {
+                let birthday_metadata = self.get_birthday_metadata(client).await?;
+                self.set_metadata(birthday_metadata.clone()).await?;
+                return Ok(birthday_metadata.utxo_index);
+            },
+            Some(m) => m,
+        };
+
         // if it's none, we return 0 above.
         let request = FindChainSplitRequest {
             block_hashes: vec![metadata.height_hash],
@@ -635,6 +636,30 @@ where TBackend: WalletBackend + 'static
         self.peer_index += 1;
         peer
     }
+
+    async fn get_birthday_metadata(
+        &self,
+        client: &mut BaseNodeSyncRpcClient,
+    ) -> Result<ScanningMetadata, UtxoScannerError> {
+        let birthday = self.resources.db.get_wallet_birthday().await?;
+        // Calculate the unix epoch time of two days before the wallet birthday. This is to avoid any weird time zone
+        // issues
+        let epoch_time = (birthday.saturating_sub(2) as u64) * 60 * 60 * 24;
+        let block_height = client.get_height_at_time(epoch_time).await?;
+        let header = client.get_header_by_height(block_height).await?;
+        let header = BlockHeader::try_from(header).map_err(|_| UtxoScannerError::ConversionError)?;
+
+        info!(
+            target: LOG_TARGET,
+            "Fresh wallet recovery starting at Block {}", block_height
+        );
+        Ok(ScanningMetadata {
+            total_amount: Default::default(),
+            number_of_utxos: 0,
+            utxo_index: header.output_mmr_size,
+            height_hash: header.hash(),
+        })
+    }
 }
 
 pub struct UtxoScannerService<TBackend>
@@ -783,7 +808,7 @@ fn convert_response_to_transaction_outputs(
     Ok((outputs, current_utxo_index))
 }
 
-#[derive(Default, Serialize, Deserialize)]
+#[derive(Clone, Default, Serialize, Deserialize)]
 struct ScanningMetadata {
     pub total_amount: MicroTari,
     pub number_of_utxos: u64,