feat: use CipherSeed wallet birthday for recovery start point

base_layer/core/src/base_node/sync/rpc/mod.rs

@@ -90,6 +90,9 @@ pub trait BaseNodeSyncService: Send + Sync + 'static {
 
     #[rpc(method = 8)]
     async fn sync_utxos(&self, request: Request<SyncUtxosRequest>) -> Result<Streaming<SyncUtxosResponse>, RpcStatus>;
+
+    #[rpc(method = 9)]
+    async fn get_height_at_time(&self, request: Request<u64>) -> Result<Response<u64>, RpcStatus>;
 }
 
 #[cfg(feature = "base_node")]

base_layer/core/src/base_node/sync/rpc/service.rs

@@ -462,4 +462,59 @@ impl<B: BlockchainBackend + 'static> BaseNodeSyncService for BaseNodeSyncRpcServ
 
         Ok(Streaming::new(rx))
     }
+
+    async fn get_height_at_time(&self, request: Request<u64>) -> Result<Response<u64>, RpcStatus> {
+        let requested_epoch_time: u64 = request.into_message();
+
+        let tip_header = self
+            .db()
+            .fetch_tip_header()
+            .await
+            .map_err(RpcStatus::log_internal_error(LOG_TARGET))?;
+        let mut left_height = 0u64;
+        let mut right_height = tip_header.height();
+
+        while left_height <= right_height {
+            let mut mid_height = (left_height + right_height) / 2;
+
+            if mid_height == 0 {
+                return Ok(Response::new(0u64));
+            }
+            // If the two bounds are adjacent then perform the test between the right and left sides
+            if left_height == mid_height {
+                mid_height = right_height;
+            }
+
+            let mid_header = self
+                .db()
+                .fetch_header(mid_height)
+                .await
+                .map_err(RpcStatus::log_internal_error(LOG_TARGET))?
+                .ok_or_else(|| {
+                    RpcStatus::not_found(format!("Header not found during search at height {}", mid_height))
+                })?;
+            let before_mid_header = self
+                .db()
+                .fetch_header(mid_height - 1)
+                .await
+                .map_err(RpcStatus::log_internal_error(LOG_TARGET))?
+                .ok_or_else(|| {
+                    RpcStatus::not_found(format!("Header not found during search at height {}", mid_height - 1))
+                })?;
+
+            if requested_epoch_time < mid_header.timestamp.as_u64() &&
+                requested_epoch_time >= before_mid_header.timestamp.as_u64()
+            {
+                return Ok(Response::new(before_mid_header.height));
+            } else if mid_height == right_height {
+                return Ok(Response::new(right_height));
+            } else if requested_epoch_time <= mid_header.timestamp.as_u64() {
+                right_height = mid_height;
+            } else {
+                left_height = mid_height;
+            }
+        }
+
+        Ok(Response::new(0u64))
+    }
 }

base_layer/core/tests/base_node_rpc.rs

@@ -42,7 +42,7 @@
 //  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
 //  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
-use std::convert::TryFrom;
+use std::{convert::TryFrom, sync::Arc, time::Duration};
 
 use randomx_rs::RandomXFlag;
 use tempfile::{tempdir, TempDir};
@@ -61,6 +61,8 @@ use tari_core::{
         },
         rpc::{BaseNodeWalletRpcService, BaseNodeWalletService},
         state_machine_service::states::{ListeningInfo, StateInfo, StatusInfo},
+        sync::rpc::BaseNodeSyncRpcService,
+        BaseNodeSyncService,
     },
     blocks::ChainBlock,
     consensus::{ConsensusManager, ConsensusManagerBuilder, NetworkConsensus},
@@ -80,14 +82,15 @@ use tari_core::{
 };
 
 use crate::helpers::{
-    block_builders::{chain_block, create_genesis_block_with_coinbase_value},
+    block_builders::{chain_block, chain_block_with_new_coinbase, create_genesis_block_with_coinbase_value},
     nodes::{BaseNodeBuilder, NodeInterfaces},
 };
 
 mod helpers;
 
 async fn setup() -> (
     BaseNodeWalletRpcService<TempDatabase>,
+    BaseNodeSyncRpcService<TempDatabase>,
     NodeInterfaces,
     RpcRequestMock,
     ConsensusManager,
@@ -118,13 +121,15 @@ async fn setup() -> (
     });
 
     let request_mock = RpcRequestMock::new(base_node.comms.peer_manager());
-    let service = BaseNodeWalletRpcService::new(
+    let wallet_service = BaseNodeWalletRpcService::new(
         base_node.blockchain_db.clone().into(),
         base_node.mempool_handle.clone(),
         base_node.state_machine_handle.clone(),
     );
+    let base_node_service = BaseNodeSyncRpcService::new(base_node.blockchain_db.clone().into());
     (
-        service,
+        wallet_service,
+        base_node_service,
         base_node,
         request_mock,
         consensus_manager,
@@ -138,7 +143,7 @@ async fn setup() -> (
 #[allow(clippy::identity_op)]
 async fn test_base_node_wallet_rpc() {
     // Testing the submit_transaction() and transaction_query() rpc calls
-    let (service, mut base_node, request_mock, consensus_manager, block0, utxo0, _temp_dir) = setup().await;
+    let (service, _, mut base_node, request_mock, consensus_manager, block0, utxo0, _temp_dir) = setup().await;
 
     let (txs1, utxos1) = schema_to_transaction(&[txn_schema!(from: vec![utxo0.clone()], to: vec![1 * T, 1 * T])]);
     let tx1 = (*txs1[0]).clone();
@@ -290,3 +295,64 @@ async fn test_base_node_wallet_rpc() {
             .any(|u| u.as_transaction_output(&factories).unwrap().commitment == output.commitment));
     }
 }
+
+#[tokio::test]
+async fn test_get_height_at_time() {
+    let factories = CryptoFactories::default();
+
+    let (_, service, base_node, request_mock, consensus_manager, block0, _utxo0, _temp_dir) = setup().await;
+
+    let mut prev_block = block0.clone();
+    let mut times = Vec::new();
+    times.push(prev_block.header().timestamp);
+    for _ in 0..10 {
+        tokio::time::sleep(Duration::from_secs(2)).await;
+        let new_block = base_node
+            .blockchain_db
+            .prepare_new_block(chain_block_with_new_coinbase(&prev_block, vec![], &consensus_manager, &factories).0)
+            .unwrap();
+
+        prev_block = base_node
+            .blockchain_db
+            .add_block(Arc::new(new_block))
+            .unwrap()
+            .assert_added();
+        times.push(prev_block.header().timestamp);
+    }
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64() - 100);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[0].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 0);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 7);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64() - 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 6);
+
+    let req = request_mock.request_with_context(Default::default(), times[7].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 7);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64());
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 10);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64() - 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 9);
+
+    let req = request_mock.request_with_context(Default::default(), times[10].as_u64() + 1);
+    let resp = service.get_height_at_time(req).await.unwrap().into_message();
+    assert_eq!(resp, 10);
+}

base_layer/key_manager/src/cipher_seed.rs

@@ -85,7 +85,7 @@ pub const CIPHER_SEED_MAC_BYTES: usize = 5;
 pub struct CipherSeed {
     version: u8,
     birthday: u16,
-    pub entropy: [u8; CIPHER_SEED_ENTROPY_BYTES],
+    entropy: [u8; CIPHER_SEED_ENTROPY_BYTES],
     salt: [u8; CIPHER_SEED_SALT_BYTES],
 }
 
@@ -108,7 +108,7 @@ impl CipherSeed {
 
     pub fn encipher(&self, passphrase: Option<String>) -> Result<Vec<u8>, KeyManagerError> {
         let mut plaintext = self.birthday.to_le_bytes().to_vec();
-        plaintext.append(&mut self.entropy.clone().to_vec());
+        plaintext.append(&mut self.entropy().clone().to_vec());
 
         let passphrase = passphrase.unwrap_or_else(|| DEFAULT_CIPHER_SEED_PASSPHRASE.to_string());
 
@@ -236,6 +236,14 @@ impl CipherSeed {
 
         Ok(())
     }
+
+    pub fn entropy(&self) -> [u8; CIPHER_SEED_ENTROPY_BYTES] {
+        self.entropy
+    }
+
+    pub fn birthday(&self) -> u16 {
+        self.birthday
+    }
 }
 
 impl Drop for CipherSeed {

base_layer/key_manager/src/key_manager.rs

@@ -74,7 +74,7 @@ where
 
     /// Derive a new private key from master key: derived_key=SHA256(master_key||branch_seed||index)
     pub fn derive_key(&self, key_index: u64) -> Result<DerivedKey<K>, ByteArrayError> {
-        let concatenated = format!("{}{}", self.seed.entropy.to_vec().to_hex(), key_index.to_string());
+        let concatenated = format!("{}{}", self.seed.entropy().to_vec().to_hex(), key_index.to_string());
         match K::from_bytes(D::digest(&concatenated.into_bytes()).as_slice()) {
             Ok(k) => Ok(DerivedKey { k, key_index }),
             Err(e) => Err(e),

base_layer/wallet/src/storage/database.rs

@@ -55,6 +55,7 @@ pub enum DbKey {
     MasterSeed,
     PassphraseHash,
     EncryptionSalt,
+    WalletBirthday,
 }
 
 pub enum DbValue {
@@ -67,6 +68,7 @@ pub enum DbValue {
     MasterSeed(CipherSeed),
     PassphraseHash(String),
     EncryptionSalt(String),
+    WalletBirthday(String),
 }
 
 #[derive(Clone)]
@@ -306,6 +308,22 @@ where T: WalletBackend + 'static
         .map_err(|err| WalletStorageError::BlockingTaskSpawnError(err.to_string()))??;
         Ok(c)
     }
+
+    pub async fn get_wallet_birthday(&self) -> Result<u16, WalletStorageError> {
+        let db_clone = self.db.clone();
+
+        let result = tokio::task::spawn_blocking(move || match db_clone.fetch(&DbKey::WalletBirthday) {
+            Ok(None) => Err(WalletStorageError::ValueNotFound(DbKey::WalletBirthday)),
+            Ok(Some(DbValue::WalletBirthday(b))) => Ok(b
+                .parse::<u16>()
+                .map_err(|_| WalletStorageError::ConversionError("Could not parse wallet birthday".to_string()))?),
+            Ok(Some(other)) => unexpected_result(DbKey::WalletBirthday, other),
+            Err(e) => log_error(DbKey::WalletBirthday, e),
+        })
+        .await
+        .map_err(|err| WalletStorageError::BlockingTaskSpawnError(err.to_string()))??;
+        Ok(result)
+    }
 }
 
 impl Display for DbKey {
@@ -319,6 +337,7 @@ impl Display for DbKey {
             DbKey::BaseNodeChainMetadata => f.write_str(&"Last seen Chain metadata from base node".to_string()),
             DbKey::PassphraseHash => f.write_str(&"PassphraseHash".to_string()),
             DbKey::EncryptionSalt => f.write_str(&"EncryptionSalt".to_string()),
+            DbKey::WalletBirthday => f.write_str(&"WalletBirthday".to_string()),
         }
     }
 }
@@ -335,6 +354,7 @@ impl Display for DbValue {
             DbValue::BaseNodeChainMetadata(v) => f.write_str(&format!("Last seen Chain metadata from base node:{}", v)),
             DbValue::PassphraseHash(h) => f.write_str(&format!("PassphraseHash: {}", h)),
             DbValue::EncryptionSalt(s) => f.write_str(&format!("EncryptionSalt: {}", s)),
+            DbValue::WalletBirthday(b) => f.write_str(&format!("WalletBirthday: {}", b)),
         }
     }
 }

base_layer/wallet/src/storage/sqlite_db.rs

@@ -83,7 +83,9 @@ impl WalletSqliteDatabase {
         match cipher.as_ref() {
             None => {
                 let seed_bytes = seed.encipher(None)?;
+                let birthday = seed.birthday();
                 WalletSettingSql::new(DbKey::MasterSeed.to_string(), seed_bytes.to_hex()).set(conn)?;
+                WalletSettingSql::new(DbKey::WalletBirthday.to_string(), birthday.to_string()).set(conn)?;
             },
             Some(cipher) => {
                 let seed_bytes = seed.encipher(None)?;
@@ -305,6 +307,9 @@ impl WalletSqliteDatabase {
             DbKey::EncryptionSalt => {
                 return Err(WalletStorageError::OperationNotSupported);
             },
+            DbKey::WalletBirthday => {
+                return Err(WalletStorageError::OperationNotSupported);
+            },
         };
         if start.elapsed().as_millis() > 0 {
             trace!(
@@ -346,6 +351,7 @@ impl WalletBackend for WalletSqliteDatabase {
             DbKey::BaseNodeChainMetadata => self.get_chain_metadata(&conn)?.map(DbValue::BaseNodeChainMetadata),
             DbKey::PassphraseHash => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::PassphraseHash),
             DbKey::EncryptionSalt => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::EncryptionSalt),
+            DbKey::WalletBirthday => WalletSettingSql::get(key.to_string(), &conn)?.map(DbValue::WalletBirthday),
         };
         if start.elapsed().as_millis() > 0 {
             trace!(

base_layer/wallet/src/utxo_scanner_service/utxo_scanning.rs

@@ -338,14 +338,15 @@ where TBackend: WalletBackend + 'static
     }
 
     async fn get_start_utxo_mmr_pos(&self, client: &mut BaseNodeSyncRpcClient) -> Result<u64, UtxoScannerError> {
-        let metadata = self.get_metadata().await?.unwrap_or_default();
-        if metadata.height_hash.is_empty() {
-            // Set a value in here so that if the recovery fails on the genesis block the client will know a
-            // recover was started. Important on Console wallet that otherwise makes this decision based on the
-            // presence of the data file
-            self.set_metadata(metadata).await?;
-            return Ok(0);
-        }
+        let metadata = match self.get_metadata().await? {
+            None => {
+                let birthday_metadata = self.get_birthday_metadata(client).await?;
+                self.set_metadata(birthday_metadata.clone()).await?;
+                return Ok(birthday_metadata.utxo_index);
+            },
+            Some(m) => m,
+        };
+
         // if it's none, we return 0 above.
         let request = FindChainSplitRequest {
             block_hashes: vec![metadata.height_hash],
@@ -635,6 +636,30 @@ where TBackend: WalletBackend + 'static
         self.peer_index += 1;
         peer
     }
+
+    async fn get_birthday_metadata(
+        &self,
+        client: &mut BaseNodeSyncRpcClient,
+    ) -> Result<ScanningMetadata, UtxoScannerError> {
+        let birthday = self.resources.db.get_wallet_birthday().await?;
+        // Calculate the unix epoch time of two days before the wallet birthday. This is to avoid any weird time zone
+        // issues
+        let epoch_time = (birthday.saturating_sub(2) as u64) * 60 * 60 * 24;
+        let block_height = client.get_height_at_time(epoch_time).await?;
+        let header = client.get_header_by_height(block_height).await?;
+        let header = BlockHeader::try_from(header).map_err(|_| UtxoScannerError::ConversionError)?;
+
+        info!(
+            target: LOG_TARGET,
+            "Fresh wallet recovery starting at Block {}", block_height
+        );
+        Ok(ScanningMetadata {
+            total_amount: Default::default(),
+            number_of_utxos: 0,
+            utxo_index: header.output_mmr_size,
+            height_hash: header.hash(),
+        })
+    }
 }
 
 pub struct UtxoScannerService<TBackend>
@@ -783,7 +808,7 @@ fn convert_response_to_transaction_outputs(
     Ok((outputs, current_utxo_index))
 }
 
-#[derive(Default, Serialize, Deserialize)]
+#[derive(Clone, Default, Serialize, Deserialize)]
 struct ScanningMetadata {
     pub total_amount: MicroTari,
     pub number_of_utxos: u64,