Zingo sync scan tasks

libtonode-tests/tests/sync.rs

@@ -23,7 +23,7 @@ async fn sync_mainnet_test() {
     let mut lightclient = LightClient::create_from_wallet_base_async(
         WalletBase::from_string(HOSPITAL_MUSEUM_SEED.to_string()),
         &config,
-        2_590_000,
+        2_611_700,
         true,
     )
     .await
@@ -34,7 +34,10 @@ async fn sync_mainnet_test() {
     sync(client, &config.chain, &mut lightclient.wallet)
         .await
         .unwrap();
+
+    dbg!(lightclient.wallet.wallet_blocks());
 }
+
 #[tokio::test]
 async fn sync_test() {
     tracing_subscriber::fmt().init();
@@ -52,4 +55,6 @@ async fn sync_test() {
     )
     .await
     .unwrap();
+
+    dbg!(recipient.wallet.wallet_blocks());
 }

zingo-sync/Cargo.toml

@@ -11,11 +11,11 @@ zingo-netutils = { path = "../zingo-netutils" }
 zcash_client_backend.workspace = true
 zcash_primitives.workspace = true
 zcash_note_encryption.workspace = true
+zcash_keys.workspace = true
 sapling-crypto.workspace = true
 orchard.workspace = true
-
-# Commitment tree
 incrementalmerkletree.workspace = true
+shardtree.workspace = true
 
 # Async
 futures.workspace = true

zingo-sync/src/interface.rs

@@ -1,18 +1,83 @@
 //! Traits for interfacing a wallet with the sync engine
 
-use std::collections::HashMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fmt::Debug;
 
 use zcash_client_backend::keys::UnifiedFullViewingKey;
+use zcash_primitives::consensus::BlockHeight;
 use zcash_primitives::zip32::AccountId;
 
+use crate::primitives::{NullifierMap, WalletBlock};
+use crate::witness::{ShardTreeData, ShardTrees};
+
 /// Temporary dump for all neccessary wallet functionality for PoC
 pub trait SyncWallet {
     /// Errors associated with interfacing the sync engine with wallet data
     type Error: Debug;
 
+    /// Returns block height wallet was created
+    fn get_birthday(&self) -> BlockHeight;
+
     /// Returns all unified full viewing keys known to this wallet.
     fn get_unified_full_viewing_keys(
         &self,
     ) -> Result<HashMap<AccountId, UnifiedFullViewingKey>, Self::Error>;
 }
+
+/// Trait for interfacing sync engine [`crate::primitives::WalletBlock`] with wallet data
+pub trait SyncBlocks: SyncWallet {
+    // TODO: add method to get wallet data for writing defualt implementations on other methods
+
+    /// Get a stored wallet compact block from wallet data by block height
+    /// Must return error if block is not found
+    fn get_wallet_block(&self, block_height: BlockHeight) -> Result<WalletBlock, Self::Error>;
+
+    /// Append wallet compact blocks to wallet data
+    fn append_wallet_blocks(
+        &mut self,
+        wallet_blocks: BTreeMap<BlockHeight, WalletBlock>,
+    ) -> Result<(), Self::Error>;
+}
+
+/// Trait for interfacing nullifiers with wallet data
+pub trait SyncNullifiers: SyncWallet {
+    // TODO: add method to get wallet data for writing defualt implementations on other methods
+
+    /// Append nullifiers to wallet data
+    fn append_nullifiers(&mut self, nullifier_map: NullifierMap) -> Result<(), Self::Error>;
+}
+
+/// Trait for interfacing shard tree data with wallet data
+pub trait SyncShardTrees: SyncWallet {
+    /// Get mutable reference to shard trees
+    fn get_shard_trees_mut(&mut self) -> Result<&mut ShardTrees, Self::Error>;
+
+    /// Update wallet data with shard tree data
+    fn update_shard_trees(&mut self, shard_tree_data: ShardTreeData) -> Result<(), Self::Error> {
+        let ShardTreeData {
+            sapling_initial_position,
+            orchard_initial_position,
+            sapling_leaves_and_retentions,
+            orchard_leaves_and_retentions,
+        } = shard_tree_data;
+
+        self.get_shard_trees_mut()
+            .unwrap()
+            .sapling_mut()
+            .batch_insert(
+                sapling_initial_position,
+                sapling_leaves_and_retentions.into_iter(),
+            )
+            .unwrap();
+        self.get_shard_trees_mut()
+            .unwrap()
+            .orchard_mut()
+            .batch_insert(
+                orchard_initial_position,
+                orchard_leaves_and_retentions.into_iter(),
+            )
+            .unwrap();
+
+        Ok(())
+    }
+}

zingo-sync/src/keys.rs

@@ -0,0 +1,160 @@
+//! Copied and modified from LRZ due to thread safety limitations and missing OVK
+
+use std::collections::HashMap;
+
+use getset::Getters;
+use incrementalmerkletree::Position;
+use orchard::{
+    keys::{FullViewingKey, IncomingViewingKey, Scope},
+    note_encryption::OrchardDomain,
+};
+use sapling_crypto::{
+    self as sapling, note_encryption::SaplingDomain, NullifierDerivingKey, SaplingIvk,
+};
+use zcash_keys::keys::UnifiedFullViewingKey;
+use zcash_note_encryption::Domain;
+
+pub(crate) type KeyId = (zcash_primitives::zip32::AccountId, Scope);
+
+/// A key that can be used to perform trial decryption and nullifier
+/// computation for a [`CompactSaplingOutput`] or [`CompactOrchardAction`].
+pub trait ScanningKeyOps<D: Domain, Nf> {
+    /// Prepare the key for use in batch trial decryption.
+    fn prepare(&self) -> D::IncomingViewingKey;
+
+    /// Returns the account identifier for this key. An account identifier corresponds
+    /// to at most a single unified spending key's worth of spend authority, such that
+    /// both received notes and change spendable by that spending authority will be
+    /// interpreted as belonging to that account.
+    fn account_id(&self) -> &zcash_primitives::zip32::AccountId;
+
+    /// Returns the [`zip32::Scope`] for which this key was derived, if known.
+    fn key_scope(&self) -> Option<Scope>;
+
+    /// Produces the nullifier for the specified note and witness, if possible.
+    ///
+    /// IVK-based implementations of this trait cannot successfully derive
+    /// nullifiers, in which this function will always return `None`.
+    fn nf(&self, note: &D::Note, note_position: Position) -> Option<Nf>;
+}
+impl<D: Domain, Nf, K: ScanningKeyOps<D, Nf>> ScanningKeyOps<D, Nf> for &K {
+    fn prepare(&self) -> D::IncomingViewingKey {
+        (*self).prepare()
+    }
+
+    fn account_id(&self) -> &zcash_primitives::zip32::AccountId {
+        (*self).account_id()
+    }
+
+    fn key_scope(&self) -> Option<Scope> {
+        (*self).key_scope()
+    }
+
+    fn nf(&self, note: &D::Note, note_position: Position) -> Option<Nf> {
+        (*self).nf(note, note_position)
+    }
+}
+
+pub(crate) struct ScanningKey<Ivk, Nk> {
+    key_id: KeyId,
+    ivk: Ivk,
+    // TODO: Ovk
+    nk: Option<Nk>,
+}
+
+impl ScanningKeyOps<SaplingDomain, sapling::Nullifier>
+    for ScanningKey<SaplingIvk, NullifierDerivingKey>
+{
+    fn prepare(&self) -> sapling::note_encryption::PreparedIncomingViewingKey {
+        sapling_crypto::note_encryption::PreparedIncomingViewingKey::new(&self.ivk)
+    }
+
+    fn nf(&self, note: &sapling::Note, position: Position) -> Option<sapling::Nullifier> {
+        self.nk.as_ref().map(|key| note.nf(key, position.into()))
+    }
+
+    fn account_id(&self) -> &zcash_primitives::zip32::AccountId {
+        &self.key_id.0
+    }
+
+    fn key_scope(&self) -> Option<Scope> {
+        Some(self.key_id.1)
+    }
+}
+
+impl ScanningKeyOps<OrchardDomain, orchard::note::Nullifier>
+    for ScanningKey<IncomingViewingKey, FullViewingKey>
+{
+    fn prepare(&self) -> orchard::keys::PreparedIncomingViewingKey {
+        orchard::keys::PreparedIncomingViewingKey::new(&self.ivk)
+    }
+
+    fn nf(
+        &self,
+        note: &orchard::note::Note,
+        _position: Position,
+    ) -> Option<orchard::note::Nullifier> {
+        self.nk.as_ref().map(|key| note.nullifier(key))
+    }
+
+    fn account_id(&self) -> &zcash_primitives::zip32::AccountId {
+        &self.key_id.0
+    }
+
+    fn key_scope(&self) -> Option<Scope> {
+        Some(self.key_id.1)
+    }
+}
+
+/// A set of keys to be used in scanning for decryptable transaction outputs.
+#[derive(Getters)]
+#[getset(get = "pub(crate)")]
+pub(crate) struct ScanningKeys {
+    sapling: HashMap<KeyId, ScanningKey<SaplingIvk, NullifierDerivingKey>>,
+    orchard: HashMap<KeyId, ScanningKey<IncomingViewingKey, FullViewingKey>>,
+}
+
+impl ScanningKeys {
+    /// Constructs a [`ScanningKeys`] from an iterator of [`zcash_keys::keys::UnifiedFullViewingKey`]s,
+    /// along with the account identifiers corresponding to those UFVKs.
+    pub(crate) fn from_account_ufvks(
+        ufvks: impl IntoIterator<Item = (zcash_primitives::zip32::AccountId, UnifiedFullViewingKey)>,
+    ) -> Self {
+        #![allow(clippy::type_complexity)]
+
+        let mut sapling: HashMap<KeyId, ScanningKey<SaplingIvk, NullifierDerivingKey>> =
+            HashMap::new();
+        let mut orchard: HashMap<KeyId, ScanningKey<IncomingViewingKey, FullViewingKey>> =
+            HashMap::new();
+
+        for (account_id, ufvk) in ufvks {
+            if let Some(dfvk) = ufvk.sapling() {
+                for scope in [Scope::External, Scope::Internal] {
+                    sapling.insert(
+                        (account_id, scope),
+                        ScanningKey {
+                            key_id: (account_id, scope),
+                            ivk: dfvk.to_ivk(scope),
+                            nk: Some(dfvk.to_nk(scope)),
+                        },
+                    );
+                }
+            }
+
+            if let Some(fvk) = ufvk.orchard() {
+                for scope in [Scope::External, Scope::Internal] {
+                    orchard.insert(
+                        (account_id, scope),
+                        ScanningKey {
+                            key_id: (account_id, scope),
+                            ivk: fvk.to_ivk(scope),
+                            nk: Some(fvk.clone()),
+                        },
+                    );
+                }
+            }
+        }
+
+        Self { sapling, orchard }
+    }
+}

zingo-sync/src/lib.rs

@@ -5,7 +5,9 @@
 
 pub mod client;
 pub mod interface;
+pub(crate) mod keys;
 #[allow(missing_docs)]
 pub mod primitives;
 pub(crate) mod scan;
 pub mod sync;
+pub mod witness;

zingo-sync/src/primitives.rs

@@ -1,8 +1,9 @@
 //! Module for primitive structs associated with the sync engine
 
-use std::sync::{Arc, RwLock};
+use std::{collections::BTreeMap, sync::Arc};
 
 use getset::{CopyGetters, Getters, MutGetters};
+use tokio::sync::RwLock;
 
 use zcash_client_backend::data_api::scanning::ScanRange;
 use zcash_primitives::{block::BlockHash, consensus::BlockHeight, transaction::TxId};
@@ -46,11 +47,33 @@ impl OutputId {
     }
 }
 
-/// Wallet compact block data
-#[allow(dead_code)]
-#[derive(CopyGetters)]
+/// Binary tree map of nullifiers from transaction spends or actions
+#[derive(Debug, MutGetters)]
+#[getset(get = "pub", get_mut = "pub")]
+pub struct NullifierMap {
+    sapling: BTreeMap<sapling_crypto::Nullifier, (BlockHeight, TxId)>,
+    orchard: BTreeMap<orchard::note::Nullifier, (BlockHeight, TxId)>,
+}
+
+impl NullifierMap {
+    pub fn new() -> Self {
+        Self {
+            sapling: BTreeMap::new(),
+            orchard: BTreeMap::new(),
+        }
+    }
+}
+
+impl Default for NullifierMap {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+/// Wallet block data
+#[derive(Debug, Clone, CopyGetters)]
 #[getset(get_copy = "pub")]
-pub struct WalletCompactBlock {
+pub struct WalletBlock {
     block_height: BlockHeight,
     block_hash: BlockHash,
     prev_hash: BlockHash,
@@ -61,7 +84,27 @@ pub struct WalletCompactBlock {
     orchard_commitment_tree_size: u32,
 }
 
-impl WalletCompactBlock {
+impl WalletBlock {
+    pub fn from_parts(
+        block_height: BlockHeight,
+        block_hash: BlockHash,
+        prev_hash: BlockHash,
+        time: u32,
+        txids: Vec<TxId>,
+        sapling_commitment_tree_size: u32,
+        orchard_commitment_tree_size: u32,
+    ) -> Self {
+        Self {
+            block_height,
+            block_hash,
+            prev_hash,
+            time,
+            txids,
+            sapling_commitment_tree_size,
+            orchard_commitment_tree_size,
+        }
+    }
+
     pub fn txids(&self) -> &[TxId] {
         &self.txids
     }