From 2ffb65618afb7382232a3c08a077dd1109005071 Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Thu, 11 Apr 2024 17:57:14 -0400 Subject: [PATCH 01/13] refactor(electrum): remove `RelevantTxids` and track txs in `TxGraph` This PR removes `RelevantTxids` from the electrum crate and tracks transactions in a `TxGraph`. This removes the need to separately construct a `TxGraph` after a `full_scan` or `sync`. --- crates/electrum/src/electrum_ext.rs | 244 ++++++++------------ crates/electrum/src/lib.rs | 11 +- crates/electrum/tests/test_electrum.rs | 39 +++- example-crates/example_electrum/src/main.rs | 37 +-- example-crates/wallet_electrum/src/main.rs | 13 +- 5 files changed, 159 insertions(+), 185 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 5a6c5d116..7ad2ae270 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,122 +1,16 @@ use bdk_chain::{ - bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, + bitcoin::{OutPoint, ScriptBuf, Txid}, + collections::{HashMap, HashSet}, local_chain::CheckPoint, - tx_graph::{self, TxGraph}, + tx_graph::TxGraph, Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; -use electrum_client::{Client, ElectrumApi, Error, HeaderNotification}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, - fmt::Debug, - str::FromStr, -}; +use electrum_client::{ElectrumApi, Error, HeaderNotification}; +use std::{collections::BTreeMap, fmt::Debug, str::FromStr}; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Represents updates fetched from an Electrum server, but excludes full transactions. -/// -/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to -/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to -/// fetch the full transactions from Electrum and finalize the update. -#[derive(Debug, Default, Clone)] -pub struct RelevantTxids(HashMap>); - -impl RelevantTxids { - /// Determine the full transactions that are missing from `graph`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn missing_full_txs(&self, graph: &TxGraph) -> Vec { - self.0 - .keys() - .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none()) - .cloned() - .collect() - } - - /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn into_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let new_txs = client.batch_transaction_get(&missing)?; - let mut graph = TxGraph::::new(new_txs); - for (txid, anchors) in self.0 { - for anchor in anchors { - let _ = graph.insert_anchor(txid, anchor); - } - } - Ok(graph) - } - - /// Finalizes the update by fetching `missing` txids from the `client`, where the - /// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`]. - /// - /// Refer to [`RelevantTxids`] for more details. - /// - /// **Note:** The confirmation time might not be precisely correct if there has been a reorg. - // Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to - // use it. - pub fn into_confirmation_time_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let graph = self.into_tx_graph(client, missing)?; - - let relevant_heights = { - let mut visited_heights = HashSet::new(); - graph - .all_anchors() - .iter() - .map(|(a, _)| a.confirmation_height_upper_bound()) - .filter(move |&h| visited_heights.insert(h)) - .collect::>() - }; - - let height_to_time = relevant_heights - .clone() - .into_iter() - .zip( - client - .batch_block_header(relevant_heights)? - .into_iter() - .map(|bh| bh.time as u64), - ) - .collect::>(); - - let graph_changeset = { - let old_changeset = TxGraph::default().apply_update(graph); - tx_graph::ChangeSet { - txs: old_changeset.txs, - txouts: old_changeset.txouts, - last_seen: old_changeset.last_seen, - anchors: old_changeset - .anchors - .into_iter() - .map(|(height_anchor, txid)| { - let confirmation_height = height_anchor.confirmation_height; - let confirmation_time = height_to_time[&confirmation_height]; - let time_anchor = ConfirmationTimeHeightAnchor { - anchor_block: height_anchor.anchor_block, - confirmation_height, - confirmation_time, - }; - (time_anchor, txid) - }) - .collect(), - } - }; - - let mut new_graph = TxGraph::default(); - new_graph.apply_changeset(graph_changeset); - Ok(new_graph) - } -} - /// Combination of chain and transactions updates from electrum /// /// We have to update the chain and the txids at the same time since we anchor the txids to @@ -125,25 +19,27 @@ impl RelevantTxids { pub struct ElectrumUpdate { /// Chain update pub chain_update: CheckPoint, - /// Transaction updates from electrum - pub relevant_txids: RelevantTxids, + /// Tracks electrum updates in TxGraph + pub graph_update: TxGraph, } -/// Trait to extend [`Client`] functionality. +/// Trait to extend [`electrum_client::Client`] functionality. pub trait ElectrumExt { /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and /// returns updates for [`bdk_chain`] data structures. /// /// - `prev_tip`: the most recent blockchain tip present locally /// - `keychain_spks`: keychains that we want to scan transactions for + /// - `full_txs`: [`TxGraph`] that contains all previously known transactions /// /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a /// single batch request. - fn full_scan( + fn full_scan( &self, prev_tip: CheckPoint, keychain_spks: BTreeMap>, + full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error>; @@ -153,7 +49,8 @@ pub trait ElectrumExt { /// /// - `prev_tip`: the most recent blockchain tip present locally /// - `misc_spks`: an iterator of scripts we want to sync transactions for - /// - `txids`: transactions for which we want updated [`Anchor`]s + /// - `full_txs`: [`TxGraph`] that contains all previously known transactions + /// - `txids`: transactions for which we want updated [`bdk_chain::Anchor`]s /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we /// want to include in the update /// @@ -164,21 +61,23 @@ pub trait ElectrumExt { /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync( + fn sync( &self, prev_tip: CheckPoint, misc_spks: impl IntoIterator, + full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, ) -> Result; } -impl ElectrumExt for A { - fn full_scan( +impl ElectrumExt for E { + fn full_scan( &self, prev_tip: CheckPoint, keychain_spks: BTreeMap>, + full_txs: Option<&TxGraph>, stop_gap: usize, batch_size: usize, ) -> Result<(ElectrumUpdate, BTreeMap), Error> { @@ -190,7 +89,14 @@ impl ElectrumExt for A { let (electrum_update, keychain_update) = loop { let (tip, _) = construct_update_tip(self, prev_tip.clone())?; - let mut relevant_txids = RelevantTxids::default(); + let mut tx_graph = TxGraph::::default(); + if let Some(txs) = full_txs { + let _ = + tx_graph.apply_update(txs.clone().map_anchors(|a| ConfirmationHeightAnchor { + anchor_block: a.anchor_block(), + confirmation_height: a.confirmation_height_upper_bound(), + })); + } let cps = tip .iter() .take(10) @@ -202,7 +108,7 @@ impl ElectrumExt for A { scanned_spks.append(&mut populate_with_spks( self, &cps, - &mut relevant_txids, + &mut tx_graph, &mut scanned_spks .iter() .map(|(i, (spk, _))| (i.clone(), spk.clone())), @@ -215,7 +121,7 @@ impl ElectrumExt for A { populate_with_spks( self, &cps, - &mut relevant_txids, + &mut tx_graph, keychain_spks, stop_gap, batch_size, @@ -234,6 +140,8 @@ impl ElectrumExt for A { let chain_update = tip; + let graph_update = into_confirmation_time_tx_graph(self, &tx_graph)?; + let keychain_update = request_spks .into_keys() .filter_map(|k| { @@ -248,7 +156,7 @@ impl ElectrumExt for A { break ( ElectrumUpdate { chain_update, - relevant_txids, + graph_update, }, keychain_update, ); @@ -257,10 +165,11 @@ impl ElectrumExt for A { Ok((electrum_update, keychain_update)) } - fn sync( + fn sync( &self, prev_tip: CheckPoint, misc_spks: impl IntoIterator, + full_txs: Option<&TxGraph>, txids: impl IntoIterator, outpoints: impl IntoIterator, batch_size: usize, @@ -273,6 +182,7 @@ impl ElectrumExt for A { let (mut electrum_update, _) = self.full_scan( prev_tip.clone(), [((), spk_iter)].into(), + full_txs, usize::MAX, batch_size, )?; @@ -284,10 +194,12 @@ impl ElectrumExt for A { .map(|cp| (cp.height(), cp)) .collect::>(); - populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?; - - let _txs = - populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?; + let mut tx_graph = TxGraph::::default(); + populate_with_txids(self, &cps, &mut tx_graph, txids)?; + populate_with_outpoints(self, &cps, &mut tx_graph, outpoints)?; + let _ = electrum_update + .graph_update + .apply_update(into_confirmation_time_tx_graph(self, &tx_graph)?); Ok(electrum_update) } @@ -411,10 +323,9 @@ fn determine_tx_anchor( fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, outpoints: impl IntoIterator, -) -> Result, Error> { - let mut full_txs = HashMap::new(); +) -> Result<(), Error> { for outpoint in outpoints { let txid = outpoint.txid; let tx = client.transaction_get(&txid)?; @@ -437,17 +348,19 @@ fn populate_with_outpoints( continue; } has_residing = true; - full_txs.insert(res.tx_hash, tx.clone()); + if tx_graph.get_tx(res.tx_hash).is_none() { + let _ = tx_graph.insert_tx(tx.clone()); + } } else { if has_spending { continue; } - let res_tx = match full_txs.get(&res.tx_hash) { + let res_tx = match tx_graph.get_tx(res.tx_hash) { Some(tx) => tx, None => { let res_tx = client.transaction_get(&res.tx_hash)?; - full_txs.insert(res.tx_hash, res_tx); - full_txs.get(&res.tx_hash).expect("just inserted") + let _ = tx_graph.insert_tx(res_tx); + tx_graph.get_tx(res.tx_hash).expect("just inserted") } }; has_spending = res_tx @@ -459,20 +372,18 @@ fn populate_with_outpoints( } }; - let anchor = determine_tx_anchor(cps, res.height, res.tx_hash); - let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default(); - if let Some(anchor) = anchor { - tx_entry.insert(anchor); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = tx_graph.insert_anchor(res.tx_hash, anchor); } } } - Ok(full_txs) + Ok(()) } fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, txids: impl IntoIterator, ) -> Result<(), Error> { for txid in txids { @@ -497,9 +408,11 @@ fn populate_with_txids( None => continue, }; - let tx_entry = relevant_txids.0.entry(txid).or_default(); + if tx_graph.get_tx(txid).is_none() { + let _ = tx_graph.insert_tx(tx); + } if let Some(anchor) = anchor { - tx_entry.insert(anchor); + let _ = tx_graph.insert_anchor(txid, anchor); } } Ok(()) @@ -508,7 +421,7 @@ fn populate_with_txids( fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, spks: &mut impl Iterator, stop_gap: usize, batch_size: usize, @@ -541,11 +454,50 @@ fn populate_with_spks( } for tx in spk_history { - let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default(); + let mut update = TxGraph::::default(); + + if tx_graph.get_tx(tx.tx_hash).is_none() { + let full_tx = client.transaction_get(&tx.tx_hash)?; + update = TxGraph::::new([full_tx]); + } + if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { - tx_entry.insert(anchor); + let _ = update.insert_anchor(tx.tx_hash, anchor); } + + let _ = tx_graph.apply_update(update); } } } } + +fn into_confirmation_time_tx_graph( + client: &impl ElectrumApi, + tx_graph: &TxGraph, +) -> Result, Error> { + let relevant_heights = tx_graph + .all_anchors() + .iter() + .map(|(a, _)| a.confirmation_height) + .collect::>(); + + let height_to_time = relevant_heights + .clone() + .into_iter() + .zip( + client + .batch_block_header(relevant_heights)? + .into_iter() + .map(|bh| bh.time as u64), + ) + .collect::>(); + + let new_graph = tx_graph + .clone() + .map_anchors(|a| ConfirmationTimeHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + confirmation_time: height_to_time[&a.confirmation_height], + }); + Ok(new_graph) +} diff --git a/crates/electrum/src/lib.rs b/crates/electrum/src/lib.rs index 87c0e4618..f645653e4 100644 --- a/crates/electrum/src/lib.rs +++ b/crates/electrum/src/lib.rs @@ -7,19 +7,10 @@ //! keychain where the range of possibly used scripts is not known. In this case it is necessary to //! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a //! sync or full scan the user receives relevant blockchain data and output updates for -//! [`bdk_chain`] including [`RelevantTxids`]. -//! -//! The [`RelevantTxids`] only includes `txid`s and not full transactions. The caller is responsible -//! for obtaining full transactions before applying new data to their [`bdk_chain`]. This can be -//! done with these steps: -//! -//! 1. Determine which full transactions are missing. Use [`RelevantTxids::missing_full_txs`]. -//! -//! 2. Obtaining the full transactions. To do this via electrum use [`ElectrumApi::batch_transaction_get`]. +//! [`bdk_chain`] including [`bdk_chain::TxGraph`], which includes `txid`s and full transactions. //! //! Refer to [`example_electrum`] for a complete example. //! -//! [`ElectrumApi::batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get //! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum #![warn(missing_docs)] diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 8de593c75..b48ee34cb 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -62,11 +62,16 @@ fn scan_detects_confirmed_tx() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -128,11 +133,16 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track.clone()], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -158,11 +168,16 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, - } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; + graph_update, + } = client.sync::( + recv_chain.tip(), + [spk_to_track.clone()], + Some(recv_graph.graph()), + None, + None, + 5, + )?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 91c3bc63d..a482e1b31 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -181,7 +181,13 @@ fn main() -> anyhow::Result<()> { }; client - .full_scan(tip, keychain_spks, stop_gap, scan_options.batch_size) + .full_scan::<_, ConfirmationHeightAnchor>( + tip, + keychain_spks, + Some(graph.lock().unwrap().graph()), + stop_gap, + scan_options.batch_size, + ) .context("scanning the blockchain")? } ElectrumCommands::Sync { @@ -274,14 +280,20 @@ fn main() -> anyhow::Result<()> { })); } - let tip = chain.tip(); + let electrum_update = client + .sync::( + chain.tip(), + spks, + Some(graph.graph()), + txids, + outpoints, + scan_options.batch_size, + ) + .context("scanning the blockchain")?; // drop lock on graph and chain drop((graph, chain)); - let electrum_update = client - .sync(tip, spks, txids, outpoints, scan_options.batch_size) - .context("scanning the blockchain")?; (electrum_update, BTreeMap::new()) } }; @@ -289,17 +301,11 @@ fn main() -> anyhow::Result<()> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, ) = response; - let missing_txids = { - let graph = &*graph.lock().unwrap(); - relevant_txids.missing_full_txs(graph.graph()) - }; - - let mut graph_update = relevant_txids.into_tx_graph(&client, missing_txids)?; let now = std::time::UNIX_EPOCH .elapsed() .expect("must get time") @@ -320,7 +326,12 @@ fn main() -> anyhow::Result<()> { indexer, ..Default::default() }); - changeset.append(graph.apply_update(graph_update)); + changeset.append(graph.apply_update(graph_update.map_anchors(|a| { + ConfirmationHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + } + }))); changeset }; diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index e2c5fd9fd..8b4cc5592 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -7,6 +7,7 @@ use std::io::Write; use std::str::FromStr; use bdk::bitcoin::{Address, Amount}; +use bdk::chain::ConfirmationTimeHeightAnchor; use bdk::wallet::Update; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; @@ -58,15 +59,19 @@ fn main() -> Result<(), anyhow::Error> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, - ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?; + ) = client.full_scan::<_, ConfirmationTimeHeightAnchor>( + prev_tip, + keychain_spks, + Some(wallet.as_ref()), + STOP_GAP, + BATCH_SIZE, + )?; println!(); - let missing = relevant_txids.missing_full_txs(wallet.as_ref()); - let mut graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); let _ = graph_update.update_last_seen_unconfirmed(now); From e3cfb84898cfa79d4903cf276fc69ffb0605b4d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 30 Apr 2024 14:49:03 +0800 Subject: [PATCH 02/13] feat(chain): `TxGraph::insert_tx` reuses `Arc` When we insert a transaction that is already wrapped in `Arc`, we should reuse the `Arc`. --- crates/chain/src/tx_graph.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/chain/src/tx_graph.rs b/crates/chain/src/tx_graph.rs index cf5148554..d097b8fa5 100644 --- a/crates/chain/src/tx_graph.rs +++ b/crates/chain/src/tx_graph.rs @@ -516,12 +516,12 @@ impl TxGraph { /// Inserts the given transaction into [`TxGraph`]. /// /// The [`ChangeSet`] returned will be empty if `tx` already exists. - pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet { + pub fn insert_tx>>(&mut self, tx: T) -> ChangeSet { + let tx = tx.into(); let mut update = Self::default(); - update.txs.insert( - tx.txid(), - (TxNodeInternal::Whole(tx.into()), BTreeSet::new(), 0), - ); + update + .txs + .insert(tx.txid(), (TxNodeInternal::Whole(tx), BTreeSet::new(), 0)); self.apply_update(update) } From 721bb7f519131ca295a00efa2d242b4923e2bddd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 1 May 2024 21:34:09 +0800 Subject: [PATCH 03/13] fix(chain): Make `Anchor` type in `FullScanResult` generic --- crates/chain/src/spk_client.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/chain/src/spk_client.rs b/crates/chain/src/spk_client.rs index 6eb32da71..5b2366e92 100644 --- a/crates/chain/src/spk_client.rs +++ b/crates/chain/src/spk_client.rs @@ -316,9 +316,9 @@ impl FullScanRequest { /// Data returned from a spk-based blockchain client full scan. /// /// See also [`FullScanRequest`]. -pub struct FullScanResult { +pub struct FullScanResult { /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain). - pub graph_update: TxGraph, + pub graph_update: TxGraph, /// The update to apply to the receiving [`TxGraph`]. pub chain_update: CheckPoint, /// Last active indices for the corresponding keychains (`K`). From 58f27b38eb2093bb9b715b7e0ebd1619ecad74ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 1 May 2024 16:24:21 +0800 Subject: [PATCH 04/13] feat(chain): introduce `TxCache` to `SyncRequest` and `FullScanRequest` This transaction cache can be provided so the chain-source can avoid re-fetching transactions. --- crates/chain/src/spk_client.rs | 67 ++++++++++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/crates/chain/src/spk_client.rs b/crates/chain/src/spk_client.rs index 5b2366e92..19813c560 100644 --- a/crates/chain/src/spk_client.rs +++ b/crates/chain/src/spk_client.rs @@ -1,11 +1,18 @@ //! Helper types for spk-based blockchain clients. +use crate::{ + collections::{BTreeMap, HashMap}, + local_chain::CheckPoint, + ConfirmationTimeHeightAnchor, TxGraph, +}; +use alloc::{boxed::Box, sync::Arc, vec::Vec}; +use bitcoin::{OutPoint, Script, ScriptBuf, Transaction, Txid}; use core::{fmt::Debug, marker::PhantomData, ops::RangeBounds}; -use alloc::{boxed::Box, collections::BTreeMap, vec::Vec}; -use bitcoin::{OutPoint, Script, ScriptBuf, Txid}; - -use crate::{local_chain::CheckPoint, ConfirmationTimeHeightAnchor, TxGraph}; +/// A cache of [`Arc`]-wrapped full transactions, identified by their [`Txid`]s. +/// +/// This is used by the chain-source to avoid re-fetching full transactions. +pub type TxCache = HashMap>; /// Data required to perform a spk-based blockchain client sync. /// @@ -17,6 +24,8 @@ pub struct SyncRequest { /// /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip pub chain_tip: CheckPoint, + /// Cache of full transactions, so the chain-source can avoid re-fetching. + pub tx_cache: TxCache, /// Transactions that spend from or to these indexed script pubkeys. pub spks: Box + Send>, /// Transactions with these txids. @@ -30,12 +39,36 @@ impl SyncRequest { pub fn from_chain_tip(cp: CheckPoint) -> Self { Self { chain_tip: cp, + tx_cache: TxCache::new(), spks: Box::new(core::iter::empty()), txids: Box::new(core::iter::empty()), outpoints: Box::new(core::iter::empty()), } } + /// Add to the [`TxCache`] held by the request. + /// + /// This consumes the [`SyncRequest`] and returns the updated one. + #[must_use] + pub fn cache_txs(mut self, full_txs: impl IntoIterator) -> Self + where + T: Into>, + { + self.tx_cache = full_txs + .into_iter() + .map(|(txid, tx)| (txid, tx.into())) + .collect(); + self + } + + /// Add all transactions from [`TxGraph`] into the [`TxCache`]. + /// + /// This consumes the [`SyncRequest`] and returns the updated one. + #[must_use] + pub fn cache_graph_txs(self, graph: &TxGraph) -> Self { + self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx))) + } + /// Set the [`Script`]s that will be synced against. /// /// This consumes the [`SyncRequest`] and returns the updated one. @@ -194,6 +227,8 @@ pub struct FullScanRequest { /// /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip pub chain_tip: CheckPoint, + /// Cache of full transactions, so the chain-source can avoid re-fetching. + pub tx_cache: TxCache, /// Iterators of script pubkeys indexed by the keychain index. pub spks_by_keychain: BTreeMap + Send>>, } @@ -204,10 +239,34 @@ impl FullScanRequest { pub fn from_chain_tip(chain_tip: CheckPoint) -> Self { Self { chain_tip, + tx_cache: TxCache::new(), spks_by_keychain: BTreeMap::new(), } } + /// Add to the [`TxCache`] held by the request. + /// + /// This consumes the [`SyncRequest`] and returns the updated one. + #[must_use] + pub fn cache_txs(mut self, full_txs: impl IntoIterator) -> Self + where + T: Into>, + { + self.tx_cache = full_txs + .into_iter() + .map(|(txid, tx)| (txid, tx.into())) + .collect(); + self + } + + /// Add all transactions from [`TxGraph`] into the [`TxCache`]. + /// + /// This consumes the [`SyncRequest`] and returns the updated one. + #[must_use] + pub fn cache_graph_txs(self, graph: &TxGraph) -> Self { + self.cache_txs(graph.full_txs().map(|tx_node| (tx_node.txid, tx_node.tx))) + } + /// Construct a new [`FullScanRequest`] from a given `chain_tip` and `index`. /// /// Unbounded script pubkey iterators for each keychain (`K`) are extracted using From 653e4fed6d16698bc5859c1e4afdcee7b3d83dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Wed, 1 May 2024 16:27:36 +0800 Subject: [PATCH 05/13] feat(wallet): cache txs when constructing full-scan/sync requests --- crates/bdk/src/wallet/mod.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/bdk/src/wallet/mod.rs b/crates/bdk/src/wallet/mod.rs index 4074c4310..146d4677e 100644 --- a/crates/bdk/src/wallet/mod.rs +++ b/crates/bdk/src/wallet/mod.rs @@ -2565,6 +2565,7 @@ impl Wallet { /// start a blockchain sync with a spk based blockchain client. pub fn start_sync_with_revealed_spks(&self) -> SyncRequest { SyncRequest::from_chain_tip(self.chain.tip()) + .cache_graph_txs(self.tx_graph()) .populate_with_revealed_spks(&self.indexed_graph.index, ..) } @@ -2578,6 +2579,7 @@ impl Wallet { /// in which the list of used scripts is not known. pub fn start_full_scan(&self) -> FullScanRequest { FullScanRequest::from_keychain_txout_index(self.chain.tip(), &self.indexed_graph.index) + .cache_graph_txs(self.tx_graph()) } } From a6fdfb2ae4caa1cdd23aa5e5ffaf02716473a98e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 30 Apr 2024 14:50:21 +0800 Subject: [PATCH 06/13] feat(electrum)!: use new sync/full-scan structs for `ElectrumExt` `ElectrumResultExt` trait is also introduced that adds methods which can convert the `Anchor` type for the update `TxGraph`. We also make use of the new `TxCache` fields in `SyncRequest`/`FullScanRequest`. This way, we can avoid re-fetching full transactions from Electrum if not needed. Examples and tests are updated to use the new `ElectrumExt` API. --- crates/electrum/Cargo.toml | 2 +- crates/electrum/src/electrum_ext.rs | 294 +++++++++++--------- crates/electrum/tests/test_electrum.rs | 71 ++--- example-crates/example_electrum/src/main.rs | 191 ++++++------- example-crates/wallet_electrum/src/main.rs | 61 ++-- 5 files changed, 289 insertions(+), 330 deletions(-) diff --git a/crates/electrum/Cargo.toml b/crates/electrum/Cargo.toml index 2f7896f77..c34470781 100644 --- a/crates/electrum/Cargo.toml +++ b/crates/electrum/Cargo.toml @@ -12,7 +12,7 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -bdk_chain = { path = "../chain", version = "0.13.0", default-features = false } +bdk_chain = { path = "../chain", version = "0.13.0" } electrum-client = { version = "0.19" } #rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] } diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 7ad2ae270..af5963052 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,28 +1,18 @@ use bdk_chain::{ - bitcoin::{OutPoint, ScriptBuf, Txid}, - collections::{HashMap, HashSet}, + bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, + collections::{BTreeMap, HashMap, HashSet}, local_chain::CheckPoint, + spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult, TxCache}, tx_graph::TxGraph, - Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, + BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, }; +use core::str::FromStr; use electrum_client::{ElectrumApi, Error, HeaderNotification}; -use std::{collections::BTreeMap, fmt::Debug, str::FromStr}; +use std::sync::Arc; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Combination of chain and transactions updates from electrum -/// -/// We have to update the chain and the txids at the same time since we anchor the txids to -/// the same chain tip that we check before and after we gather the txids. -#[derive(Debug)] -pub struct ElectrumUpdate { - /// Chain update - pub chain_update: CheckPoint, - /// Tracks electrum updates in TxGraph - pub graph_update: TxGraph, -} - /// Trait to extend [`electrum_client::Client`] functionality. pub trait ElectrumExt { /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and @@ -35,14 +25,12 @@ pub trait ElectrumExt { /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a /// single batch request. - fn full_scan( + fn full_scan( &self, - prev_tip: CheckPoint, - keychain_spks: BTreeMap>, - full_txs: Option<&TxGraph>, + request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result<(ElectrumUpdate, BTreeMap), Error>; + ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified /// and returns updates for [`bdk_chain`] data structures. @@ -61,42 +49,33 @@ pub trait ElectrumExt { /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync( + fn sync( &self, - prev_tip: CheckPoint, - misc_spks: impl IntoIterator, - full_txs: Option<&TxGraph>, - txids: impl IntoIterator, - outpoints: impl IntoIterator, + request: SyncRequest, batch_size: usize, - ) -> Result; + ) -> Result, Error>; } impl ElectrumExt for E { - fn full_scan( + fn full_scan( &self, - prev_tip: CheckPoint, - keychain_spks: BTreeMap>, - full_txs: Option<&TxGraph>, + mut request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result<(ElectrumUpdate, BTreeMap), Error> { - let mut request_spks = keychain_spks - .into_iter() - .map(|(k, s)| (k, s.into_iter())) - .collect::>(); + ) -> Result, Error> { + let mut request_spks = request.spks_by_keychain; + + // We keep track of already-scanned spks just in case a reorg happens and we need to do a + // rescan. We need to keep track of this as iterators in `keychain_spks` are "unbounded" so + // cannot be collected. In addition, we keep track of whether an spk has an active tx + // history for determining the `last_active_index`. + // * key: (keychain, spk_index) that identifies the spk. + // * val: (script_pubkey, has_tx_history). let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new(); - let (electrum_update, keychain_update) = loop { - let (tip, _) = construct_update_tip(self, prev_tip.clone())?; - let mut tx_graph = TxGraph::::default(); - if let Some(txs) = full_txs { - let _ = - tx_graph.apply_update(txs.clone().map_anchors(|a| ConfirmationHeightAnchor { - anchor_block: a.anchor_block(), - confirmation_height: a.confirmation_height_upper_bound(), - })); - } + let update = loop { + let (tip, _) = construct_update_tip(self, request.chain_tip.clone())?; + let mut graph_update = TxGraph::::default(); let cps = tip .iter() .take(10) @@ -108,7 +87,8 @@ impl ElectrumExt for E { scanned_spks.append(&mut populate_with_spks( self, &cps, - &mut tx_graph, + &mut request.tx_cache, + &mut graph_update, &mut scanned_spks .iter() .map(|(i, (spk, _))| (i.clone(), spk.clone())), @@ -121,7 +101,8 @@ impl ElectrumExt for E { populate_with_spks( self, &cps, - &mut tx_graph, + &mut request.tx_cache, + &mut graph_update, keychain_spks, stop_gap, batch_size, @@ -140,8 +121,6 @@ impl ElectrumExt for E { let chain_update = tip; - let graph_update = into_confirmation_time_tx_graph(self, &tx_graph)?; - let keychain_update = request_spks .into_keys() .filter_map(|k| { @@ -153,41 +132,29 @@ impl ElectrumExt for E { }) .collect::>(); - break ( - ElectrumUpdate { - chain_update, - graph_update, - }, - keychain_update, - ); + break FullScanResult { + graph_update, + chain_update, + last_active_indices: keychain_update, + }; }; - Ok((electrum_update, keychain_update)) + Ok(update) } - fn sync( + fn sync( &self, - prev_tip: CheckPoint, - misc_spks: impl IntoIterator, - full_txs: Option<&TxGraph>, - txids: impl IntoIterator, - outpoints: impl IntoIterator, + request: SyncRequest, batch_size: usize, - ) -> Result { - let spk_iter = misc_spks - .into_iter() - .enumerate() - .map(|(i, spk)| (i as u32, spk)); - - let (mut electrum_update, _) = self.full_scan( - prev_tip.clone(), - [((), spk_iter)].into(), - full_txs, - usize::MAX, - batch_size, - )?; - - let (tip, _) = construct_update_tip(self, prev_tip)?; + ) -> Result, Error> { + let mut tx_cache = request.tx_cache.clone(); + + let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) + .cache_txs(request.tx_cache) + .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); + let full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size)?; + + let (tip, _) = construct_update_tip(self, request.chain_tip)?; let cps = tip .iter() .take(10) @@ -195,16 +162,88 @@ impl ElectrumExt for E { .collect::>(); let mut tx_graph = TxGraph::::default(); - populate_with_txids(self, &cps, &mut tx_graph, txids)?; - populate_with_outpoints(self, &cps, &mut tx_graph, outpoints)?; - let _ = electrum_update - .graph_update - .apply_update(into_confirmation_time_tx_graph(self, &tx_graph)?); + populate_with_txids(self, &cps, &mut tx_cache, &mut tx_graph, request.txids)?; + populate_with_outpoints(self, &cps, &mut tx_cache, &mut tx_graph, request.outpoints)?; - Ok(electrum_update) + Ok(SyncResult { + chain_update: full_scan_res.chain_update, + graph_update: full_scan_res.graph_update, + }) } } +/// Trait that extends [`SyncResult`] and [`FullScanResult`] functionality. +/// +/// Currently, only a single method exists that converts the update [`TxGraph`] to have an anchor +/// type of [`ConfirmationTimeHeightAnchor`]. +pub trait ElectrumResultExt { + /// New result type with a [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`]. + type NewResult; + + /// Convert result type to have an update [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`] . + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result; +} + +impl ElectrumResultExt for FullScanResult { + type NewResult = FullScanResult; + + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result { + Ok(FullScanResult:: { + graph_update: try_into_confirmation_time_result(self.graph_update, client)?, + chain_update: self.chain_update, + last_active_indices: self.last_active_indices, + }) + } +} + +impl ElectrumResultExt for SyncResult { + type NewResult = SyncResult; + + fn try_into_confirmation_time_result( + self, + client: &impl ElectrumApi, + ) -> Result { + Ok(SyncResult { + graph_update: try_into_confirmation_time_result(self.graph_update, client)?, + chain_update: self.chain_update, + }) + } +} + +fn try_into_confirmation_time_result( + graph_update: TxGraph, + client: &impl ElectrumApi, +) -> Result, Error> { + let relevant_heights = graph_update + .all_anchors() + .iter() + .map(|(a, _)| a.confirmation_height) + .collect::>(); + + let height_to_time = relevant_heights + .clone() + .into_iter() + .zip( + client + .batch_block_header(relevant_heights)? + .into_iter() + .map(|bh| bh.time as u64), + ) + .collect::>(); + + Ok(graph_update.map_anchors(|a| ConfirmationTimeHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + confirmation_time: height_to_time[&a.confirmation_height], + })) +} + /// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`. fn construct_update_tip( client: &impl ElectrumApi, @@ -323,6 +362,7 @@ fn determine_tx_anchor( fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, + tx_cache: &mut TxCache, tx_graph: &mut TxGraph, outpoints: impl IntoIterator, ) -> Result<(), Error> { @@ -358,9 +398,9 @@ fn populate_with_outpoints( let res_tx = match tx_graph.get_tx(res.tx_hash) { Some(tx) => tx, None => { - let res_tx = client.transaction_get(&res.tx_hash)?; - let _ = tx_graph.insert_tx(res_tx); - tx_graph.get_tx(res.tx_hash).expect("just inserted") + let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; + let _ = tx_graph.insert_tx(Arc::clone(&res_tx)); + res_tx } }; has_spending = res_tx @@ -383,11 +423,12 @@ fn populate_with_outpoints( fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, - tx_graph: &mut TxGraph, + tx_cache: &mut TxCache, + graph_update: &mut TxGraph, txids: impl IntoIterator, ) -> Result<(), Error> { for txid in txids { - let tx = match client.transaction_get(&txid) { + let tx = match fetch_tx(client, tx_cache, txid) { Ok(tx) => tx, Err(electrum_client::Error::Protocol(_)) => continue, Err(other_err) => return Err(other_err), @@ -408,20 +449,36 @@ fn populate_with_txids( None => continue, }; - if tx_graph.get_tx(txid).is_none() { - let _ = tx_graph.insert_tx(tx); + if graph_update.get_tx(txid).is_none() { + // TODO: We need to be able to insert an `Arc` of a transaction. + let _ = graph_update.insert_tx(tx); } if let Some(anchor) = anchor { - let _ = tx_graph.insert_anchor(txid, anchor); + let _ = graph_update.insert_anchor(txid, anchor); } } Ok(()) } +fn fetch_tx( + client: &C, + tx_cache: &mut TxCache, + txid: Txid, +) -> Result, Error> { + use bdk_chain::collections::hash_map::Entry; + Ok(match tx_cache.entry(txid) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => entry + .insert(Arc::new(client.transaction_get(&txid)?)) + .clone(), + }) +} + fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, - tx_graph: &mut TxGraph, + tx_cache: &mut TxCache, + graph_update: &mut TxGraph, spks: &mut impl Iterator, stop_gap: usize, batch_size: usize, @@ -453,51 +510,12 @@ fn populate_with_spks( unused_spk_count = 0; } - for tx in spk_history { - let mut update = TxGraph::::default(); - - if tx_graph.get_tx(tx.tx_hash).is_none() { - let full_tx = client.transaction_get(&tx.tx_hash)?; - update = TxGraph::::new([full_tx]); + for tx_res in spk_history { + let _ = graph_update.insert_tx(fetch_tx(client, tx_cache, tx_res.tx_hash)?); + if let Some(anchor) = determine_tx_anchor(cps, tx_res.height, tx_res.tx_hash) { + let _ = graph_update.insert_anchor(tx_res.tx_hash, anchor); } - - if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { - let _ = update.insert_anchor(tx.tx_hash, anchor); - } - - let _ = tx_graph.apply_update(update); } } } } - -fn into_confirmation_time_tx_graph( - client: &impl ElectrumApi, - tx_graph: &TxGraph, -) -> Result, Error> { - let relevant_heights = tx_graph - .all_anchors() - .iter() - .map(|(a, _)| a.confirmation_height) - .collect::>(); - - let height_to_time = relevant_heights - .clone() - .into_iter() - .zip( - client - .batch_block_header(relevant_heights)? - .into_iter() - .map(|bh| bh.time as u64), - ) - .collect::>(); - - let new_graph = tx_graph - .clone() - .map_anchors(|a| ConfirmationTimeHeightAnchor { - anchor_block: a.anchor_block, - confirmation_height: a.confirmation_height, - confirmation_time: height_to_time[&a.confirmation_height], - }); - Ok(new_graph) -} diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index b48ee34cb..aa4b87933 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -2,9 +2,10 @@ use bdk_chain::{ bitcoin::{hashes::Hash, Address, Amount, ScriptBuf, WScriptHash}, keychain::Balance, local_chain::LocalChain, + spk_client::SyncRequest, ConfirmationTimeHeightAnchor, IndexedTxGraph, SpkTxOutIndex, }; -use bdk_electrum::{ElectrumExt, ElectrumUpdate}; +use bdk_electrum::{ElectrumExt, ElectrumResultExt}; use bdk_testenv::{anyhow, anyhow::Result, bitcoincore_rpc::RpcApi, TestEnv}; fn get_balance( @@ -60,22 +61,18 @@ fn scan_detects_confirmed_tx() -> Result<()> { // Sync up to tip. env.wait_until_electrum_sees_block()?; - let ElectrumUpdate { - chain_update, - graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + let update = client + .sync( + SyncRequest::from_chain_tip(recv_chain.tip()) + .chain_spks(core::iter::once(spk_to_track)), + 5, + )? + .try_into_confirmation_time_result(&client)?; let _ = recv_chain - .apply_update(chain_update) + .apply_update(update.chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; - let _ = recv_graph.apply_update(graph_update); + let _ = recv_graph.apply_update(update.graph_update); // Check to see if tx is confirmed. assert_eq!( @@ -131,25 +128,20 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { // Sync up to tip. env.wait_until_electrum_sees_block()?; - let ElectrumUpdate { - chain_update, - graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track.clone()], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + let update = client + .sync( + SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), + 5, + )? + .try_into_confirmation_time_result(&client)?; let _ = recv_chain - .apply_update(chain_update) + .apply_update(update.chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; - let _ = recv_graph.apply_update(graph_update.clone()); + let _ = recv_graph.apply_update(update.graph_update.clone()); // Retain a snapshot of all anchors before reorg process. - let initial_anchors = graph_update.all_anchors(); + let initial_anchors = update.graph_update.all_anchors(); // Check if initial balance is correct. assert_eq!( @@ -166,27 +158,22 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.reorg_empty_blocks(depth)?; env.wait_until_electrum_sees_block()?; - let ElectrumUpdate { - chain_update, - graph_update, - } = client.sync::( - recv_chain.tip(), - [spk_to_track.clone()], - Some(recv_graph.graph()), - None, - None, - 5, - )?; + let update = client + .sync( + SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), + 5, + )? + .try_into_confirmation_time_result(&client)?; let _ = recv_chain - .apply_update(chain_update) + .apply_update(update.chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; // Check to see if a new anchor is added during current reorg. - if !initial_anchors.is_superset(graph_update.all_anchors()) { + if !initial_anchors.is_superset(update.graph_update.all_anchors()) { println!("New anchor added at reorg depth {}", depth); } - let _ = recv_graph.apply_update(graph_update); + let _ = recv_graph.apply_update(update.graph_update); assert_eq!( get_balance(&recv_chain, &recv_graph)?, diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index a482e1b31..d321ded7e 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -1,19 +1,20 @@ use std::{ - collections::BTreeMap, io::{self, Write}, sync::Mutex, }; use bdk_chain::{ - bitcoin::{constants::genesis_block, Address, Network, OutPoint, Txid}, + bitcoin::{constants::genesis_block, Address, Network, Txid}, + collections::BTreeSet, indexed_tx_graph::{self, IndexedTxGraph}, keychain, local_chain::{self, LocalChain}, + spk_client::{FullScanRequest, SyncRequest}, Append, ConfirmationHeightAnchor, }; use bdk_electrum::{ electrum_client::{self, Client, ElectrumApi}, - ElectrumExt, ElectrumUpdate, + ElectrumExt, }; use example_cli::{ anyhow::{self, Context}, @@ -147,48 +148,55 @@ fn main() -> anyhow::Result<()> { let client = electrum_cmd.electrum_args().client(args.network)?; - let response = match electrum_cmd.clone() { + let (chain_update, mut graph_update, keychain_update) = match electrum_cmd.clone() { ElectrumCommands::Scan { stop_gap, scan_options, .. } => { - let (keychain_spks, tip) = { + let request = { let graph = &*graph.lock().unwrap(); let chain = &*chain.lock().unwrap(); - let keychain_spks = graph - .index - .all_unbounded_spk_iters() - .into_iter() - .map(|(keychain, iter)| { - let mut first = true; - let spk_iter = iter.inspect(move |(i, _)| { - if first { - eprint!("\nscanning {}: ", keychain); - first = false; + FullScanRequest::from_chain_tip(chain.tip()) + .cache_graph_txs(graph.graph()) + .set_spks_for_keychain( + Keychain::External, + graph + .index + .unbounded_spk_iter(&Keychain::External) + .into_iter() + .flatten(), + ) + .set_spks_for_keychain( + Keychain::Internal, + graph + .index + .unbounded_spk_iter(&Keychain::Internal) + .into_iter() + .flatten(), + ) + .inspect_spks_for_all_keychains({ + let mut once = BTreeSet::new(); + move |k, spk_i, _| { + if once.insert(k) { + eprint!("\nScanning {}: ", k); + } else { + eprint!("{} ", spk_i); } - - eprint!("{} ", i); let _ = io::stdout().flush(); - }); - (keychain, spk_iter) + } }) - .collect::>(); - - let tip = chain.tip(); - (keychain_spks, tip) }; - client - .full_scan::<_, ConfirmationHeightAnchor>( - tip, - keychain_spks, - Some(graph.lock().unwrap().graph()), - stop_gap, - scan_options.batch_size, - ) - .context("scanning the blockchain")? + let res = client + .full_scan::<_>(request, stop_gap, scan_options.batch_size) + .context("scanning the blockchain")?; + ( + res.chain_update, + res.graph_update, + Some(res.last_active_indices), + ) } ElectrumCommands::Sync { mut unused_spks, @@ -201,7 +209,6 @@ fn main() -> anyhow::Result<()> { // Get a short lock on the tracker to get the spks we're interested in let graph = graph.lock().unwrap(); let chain = chain.lock().unwrap(); - let chain_tip = chain.tip().block_id(); if !(all_spks || unused_spks || utxos || unconfirmed) { unused_spks = true; @@ -211,18 +218,20 @@ fn main() -> anyhow::Result<()> { unused_spks = false; } - let mut spks: Box> = - Box::new(core::iter::empty()); + let chain_tip = chain.tip(); + let mut request = + SyncRequest::from_chain_tip(chain_tip.clone()).cache_graph_txs(graph.graph()); + if all_spks { let all_spks = graph .index .revealed_spks(..) .map(|(k, i, spk)| (k.to_owned(), i, spk.to_owned())) .collect::>(); - spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| { - eprintln!("scanning {}:{}", k, i); + request = request.chain_spks(all_spks.into_iter().map(|(k, spk_i, spk)| { + eprintln!("scanning {}: {}", k, spk_i); spk - }))); + })); } if unused_spks { let unused_spks = graph @@ -230,82 +239,61 @@ fn main() -> anyhow::Result<()> { .unused_spks() .map(|(k, i, spk)| (k, i, spk.to_owned())) .collect::>(); - spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| { - eprintln!( - "Checking if address {} {}:{} has been used", - Address::from_script(&spk, args.network).unwrap(), - k, - i, - ); - spk - }))); + request = + request.chain_spks(unused_spks.into_iter().map(move |(k, spk_i, spk)| { + eprintln!( + "Checking if address {} {}:{} has been used", + Address::from_script(&spk, args.network).unwrap(), + k, + spk_i, + ); + spk + })); } - let mut outpoints: Box> = Box::new(core::iter::empty()); - if utxos { let init_outpoints = graph.index.outpoints(); let utxos = graph .graph() - .filter_chain_unspents(&*chain, chain_tip, init_outpoints) + .filter_chain_unspents(&*chain, chain_tip.block_id(), init_outpoints) .map(|(_, utxo)| utxo) .collect::>(); - - outpoints = Box::new( - utxos - .into_iter() - .inspect(|utxo| { - eprintln!( - "Checking if outpoint {} (value: {}) has been spent", - utxo.outpoint, utxo.txout.value - ); - }) - .map(|utxo| utxo.outpoint), - ); + request = request.chain_outpoints(utxos.into_iter().map(|utxo| { + eprintln!( + "Checking if outpoint {} (value: {}) has been spent", + utxo.outpoint, utxo.txout.value + ); + utxo.outpoint + })); }; - let mut txids: Box> = Box::new(core::iter::empty()); - if unconfirmed { let unconfirmed_txids = graph .graph() - .list_chain_txs(&*chain, chain_tip) + .list_chain_txs(&*chain, chain_tip.block_id()) .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed()) .map(|canonical_tx| canonical_tx.tx_node.txid) .collect::>(); - txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| { - eprintln!("Checking if {} is confirmed yet", txid); - })); + request = request.chain_txids( + unconfirmed_txids + .into_iter() + .inspect(|txid| eprintln!("Checking if {} is confirmed yet", txid)), + ); } - let electrum_update = client - .sync::( - chain.tip(), - spks, - Some(graph.graph()), - txids, - outpoints, - scan_options.batch_size, - ) + let res = client + .sync(request, scan_options.batch_size) .context("scanning the blockchain")?; // drop lock on graph and chain drop((graph, chain)); - (electrum_update, BTreeMap::new()) + (res.chain_update, res.graph_update, None) } }; - let ( - ElectrumUpdate { - chain_update, - mut graph_update, - }, - keychain_update, - ) = response; - let now = std::time::UNIX_EPOCH .elapsed() .expect("must get time") @@ -316,26 +304,17 @@ fn main() -> anyhow::Result<()> { let mut chain = chain.lock().unwrap(); let mut graph = graph.lock().unwrap(); - let chain = chain.apply_update(chain_update)?; - - let indexed_tx_graph = { - let mut changeset = - indexed_tx_graph::ChangeSet::::default(); - let (_, indexer) = graph.index.reveal_to_target_multi(&keychain_update); - changeset.append(indexed_tx_graph::ChangeSet { - indexer, - ..Default::default() - }); - changeset.append(graph.apply_update(graph_update.map_anchors(|a| { - ConfirmationHeightAnchor { - anchor_block: a.anchor_block, - confirmation_height: a.confirmation_height, - } - }))); - changeset - }; - - (chain, indexed_tx_graph) + let chain_changeset = chain.apply_update(chain_update)?; + + let mut indexed_tx_graph_changeset = + indexed_tx_graph::ChangeSet::::default(); + if let Some(keychain_update) = keychain_update { + let (_, keychain_changeset) = graph.index.reveal_to_target_multi(&keychain_update); + indexed_tx_graph_changeset.append(keychain_changeset.into()); + } + indexed_tx_graph_changeset.append(graph.apply_update(graph_update)); + + (chain_changeset, indexed_tx_graph_changeset) }; let mut db = db.lock().unwrap(); diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index 8b4cc5592..bdbf32120 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -3,17 +3,16 @@ const SEND_AMOUNT: Amount = Amount::from_sat(5000); const STOP_GAP: usize = 50; const BATCH_SIZE: usize = 5; -use std::io::Write; use std::str::FromStr; use bdk::bitcoin::{Address, Amount}; -use bdk::chain::ConfirmationTimeHeightAnchor; -use bdk::wallet::Update; +use bdk::chain::collections::HashSet; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; +use bdk_electrum::ElectrumResultExt; use bdk_electrum::{ electrum_client::{self, ElectrumApi}, - ElectrumExt, ElectrumUpdate, + ElectrumExt, }; use bdk_file_store::Store; @@ -39,48 +38,24 @@ fn main() -> Result<(), anyhow::Error> { print!("Syncing..."); let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?; - let prev_tip = wallet.latest_checkpoint(); - let keychain_spks = wallet - .all_unbounded_spk_iters() - .into_iter() - .map(|(k, k_spks)| { - let mut once = Some(()); - let mut stdout = std::io::stdout(); - let k_spks = k_spks - .inspect(move |(spk_i, _)| match once.take() { - Some(_) => print!("\nScanning keychain [{:?}]", k), - None => print!(" {:<3}", spk_i), - }) - .inspect(move |_| stdout.flush().expect("must flush")); - (k, k_spks) - }) - .collect(); - - let ( - ElectrumUpdate { - chain_update, - mut graph_update, - }, - keychain_update, - ) = client.full_scan::<_, ConfirmationTimeHeightAnchor>( - prev_tip, - keychain_spks, - Some(wallet.as_ref()), - STOP_GAP, - BATCH_SIZE, - )?; + let request = wallet.start_full_scan().inspect_spks_for_all_keychains({ + let mut once = HashSet::::new(); + move |k, spk_i, _| match once.insert(k) { + true => print!("\nScanning keychain [{:?}]", k), + false => print!(" {:<3}", spk_i), + } + }); - println!(); + let mut update = client + .full_scan(request, STOP_GAP, BATCH_SIZE)? + .try_into_confirmation_time_result(&client)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); - let _ = graph_update.update_last_seen_unconfirmed(now); - - let wallet_update = Update { - last_active_indices: keychain_update, - graph: graph_update, - chain: Some(chain_update), - }; - wallet.apply_update(wallet_update)?; + let _ = update.graph_update.update_last_seen_unconfirmed(now); + + println!(); + + wallet.apply_update(update)?; wallet.commit()?; let balance = wallet.get_balance(); From b1f861b932afd5e490c0814b1921b97cc2f1d912 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Sat, 4 May 2024 19:27:11 +0800 Subject: [PATCH 07/13] feat: update logging of electrum examples * Syncing with `example_electrum` now shows progress as a percentage. * Flush stdout more aggressively. --- example-crates/example_electrum/src/main.rs | 38 +++++++++++++++++---- example-crates/wallet_electrum/src/main.rs | 21 ++++++++---- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index d321ded7e..58474668f 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -180,11 +180,11 @@ fn main() -> anyhow::Result<()> { let mut once = BTreeSet::new(); move |k, spk_i, _| { if once.insert(k) { - eprint!("\nScanning {}: ", k); + eprint!("\nScanning {}: {} ", k, spk_i); } else { eprint!("{} ", spk_i); } - let _ = io::stdout().flush(); + io::stdout().flush().expect("must flush"); } }) }; @@ -229,7 +229,7 @@ fn main() -> anyhow::Result<()> { .map(|(k, i, spk)| (k.to_owned(), i, spk.to_owned())) .collect::>(); request = request.chain_spks(all_spks.into_iter().map(|(k, spk_i, spk)| { - eprintln!("scanning {}: {}", k, spk_i); + eprint!("Scanning {}: {}", k, spk_i); spk })); } @@ -241,7 +241,7 @@ fn main() -> anyhow::Result<()> { .collect::>(); request = request.chain_spks(unused_spks.into_iter().map(move |(k, spk_i, spk)| { - eprintln!( + eprint!( "Checking if address {} {}:{} has been used", Address::from_script(&spk, args.network).unwrap(), k, @@ -260,7 +260,7 @@ fn main() -> anyhow::Result<()> { .map(|(_, utxo)| utxo) .collect::>(); request = request.chain_outpoints(utxos.into_iter().map(|utxo| { - eprintln!( + eprint!( "Checking if outpoint {} (value: {}) has been spent", utxo.outpoint, utxo.txout.value ); @@ -279,10 +279,36 @@ fn main() -> anyhow::Result<()> { request = request.chain_txids( unconfirmed_txids .into_iter() - .inspect(|txid| eprintln!("Checking if {} is confirmed yet", txid)), + .inspect(|txid| eprint!("Checking if {} is confirmed yet", txid)), ); } + let total_spks = request.spks.len(); + let total_txids = request.txids.len(); + let total_ops = request.outpoints.len(); + request = request + .inspect_spks({ + let mut visited = 0; + move |_| { + visited += 1; + eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_spks as f32) + } + }) + .inspect_txids({ + let mut visited = 0; + move |_| { + visited += 1; + eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_txids as f32) + } + }) + .inspect_outpoints({ + let mut visited = 0; + move |_| { + visited += 1; + eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_ops as f32) + } + }); + let res = client .sync(request, scan_options.batch_size) .context("scanning the blockchain")?; diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index bdbf32120..e1fe00984 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -3,6 +3,7 @@ const SEND_AMOUNT: Amount = Amount::from_sat(5000); const STOP_GAP: usize = 50; const BATCH_SIZE: usize = 5; +use std::io::Write; use std::str::FromStr; use bdk::bitcoin::{Address, Amount}; @@ -38,13 +39,19 @@ fn main() -> Result<(), anyhow::Error> { print!("Syncing..."); let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?; - let request = wallet.start_full_scan().inspect_spks_for_all_keychains({ - let mut once = HashSet::::new(); - move |k, spk_i, _| match once.insert(k) { - true => print!("\nScanning keychain [{:?}]", k), - false => print!(" {:<3}", spk_i), - } - }); + let request = wallet + .start_full_scan() + .inspect_spks_for_all_keychains({ + let mut once = HashSet::::new(); + move |k, spk_i, _| { + if once.insert(k) { + print!("\nScanning keychain [{:?}]", k) + } else { + print!(" {:<3}", spk_i) + } + } + }) + .inspect_spks_for_all_keychains(|_, _, _| std::io::stdout().flush().expect("must flush")); let mut update = client .full_scan(request, STOP_GAP, BATCH_SIZE)? From 9ed33c25ea01278b0a47c8ecd5ea6fa33119a977 Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Mon, 6 May 2024 15:46:49 +0800 Subject: [PATCH 08/13] docs(electrum): fixed `full_scan`, `sync`, and crate documentation --- crates/electrum/src/electrum_ext.rs | 14 ++++---------- crates/electrum/src/lib.rs | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index af5963052..a746c52cf 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -18,9 +18,8 @@ pub trait ElectrumExt { /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and /// returns updates for [`bdk_chain`] data structures. /// - /// - `prev_tip`: the most recent blockchain tip present locally - /// - `keychain_spks`: keychains that we want to scan transactions for - /// - `full_txs`: [`TxGraph`] that contains all previously known transactions + /// - `request`: struct with data required to perform a spk-based blockchain client full scan, + /// see [`FullScanRequest`] /// /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a @@ -35,12 +34,8 @@ pub trait ElectrumExt { /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified /// and returns updates for [`bdk_chain`] data structures. /// - /// - `prev_tip`: the most recent blockchain tip present locally - /// - `misc_spks`: an iterator of scripts we want to sync transactions for - /// - `full_txs`: [`TxGraph`] that contains all previously known transactions - /// - `txids`: transactions for which we want updated [`bdk_chain::Anchor`]s - /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we - /// want to include in the update + /// - `request`: struct with data required to perform a spk-based blockchain client sync, + /// see [`SyncRequest`] /// /// `batch_size` specifies the max number of script pubkeys to request for in a single batch /// request. @@ -450,7 +445,6 @@ fn populate_with_txids( }; if graph_update.get_tx(txid).is_none() { - // TODO: We need to be able to insert an `Arc` of a transaction. let _ = graph_update.insert_tx(tx); } if let Some(anchor) = anchor { diff --git a/crates/electrum/src/lib.rs b/crates/electrum/src/lib.rs index f645653e4..eaa2405bf 100644 --- a/crates/electrum/src/lib.rs +++ b/crates/electrum/src/lib.rs @@ -7,7 +7,7 @@ //! keychain where the range of possibly used scripts is not known. In this case it is necessary to //! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a //! sync or full scan the user receives relevant blockchain data and output updates for -//! [`bdk_chain`] including [`bdk_chain::TxGraph`], which includes `txid`s and full transactions. +//! [`bdk_chain`]. //! //! Refer to [`example_electrum`] for a complete example. //! From 2945c6be88b3bf5105afeb8addff4861f0458b41 Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Mon, 6 May 2024 15:52:57 +0800 Subject: [PATCH 09/13] fix(electrum): fixed `sync` functionality --- crates/electrum/src/electrum_ext.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index a746c52cf..4c6786557 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -147,7 +147,7 @@ impl ElectrumExt for E { let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) .cache_txs(request.tx_cache) .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); - let full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size)?; + let mut full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size)?; let (tip, _) = construct_update_tip(self, request.chain_tip)?; let cps = tip @@ -156,9 +156,20 @@ impl ElectrumExt for E { .map(|cp| (cp.height(), cp)) .collect::>(); - let mut tx_graph = TxGraph::::default(); - populate_with_txids(self, &cps, &mut tx_cache, &mut tx_graph, request.txids)?; - populate_with_outpoints(self, &cps, &mut tx_cache, &mut tx_graph, request.outpoints)?; + populate_with_txids( + self, + &cps, + &mut tx_cache, + &mut full_scan_res.graph_update, + request.txids, + )?; + populate_with_outpoints( + self, + &cps, + &mut tx_cache, + &mut full_scan_res.graph_update, + request.outpoints, + )?; Ok(SyncResult { chain_update: full_scan_res.chain_update, From c0d7d60a582b939324bb48ec8c5035020ec90699 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Tue, 7 May 2024 12:43:02 +0800 Subject: [PATCH 10/13] feat(chain)!: use custom return types for `ElectrumExt` methods This is more code, but a much more elegant solution than having `ElectrumExt` methods return `SyncResult`/`FullScanResult` and having an `ElectrumResultExt` extention trait. --- crates/electrum/src/electrum_ext.rs | 93 +++++++++++---------- crates/electrum/tests/test_electrum.rs | 8 +- example-crates/example_electrum/src/main.rs | 6 +- example-crates/wallet_electrum/src/main.rs | 3 +- 4 files changed, 58 insertions(+), 52 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 4c6786557..806cabeb2 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -29,7 +29,7 @@ pub trait ElectrumExt { request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result, Error>; + ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified /// and returns updates for [`bdk_chain`] data structures. @@ -44,11 +44,7 @@ pub trait ElectrumExt { /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync( - &self, - request: SyncRequest, - batch_size: usize, - ) -> Result, Error>; + fn sync(&self, request: SyncRequest, batch_size: usize) -> Result; } impl ElectrumExt for E { @@ -57,7 +53,7 @@ impl ElectrumExt for E { mut request: FullScanRequest, stop_gap: usize, batch_size: usize, - ) -> Result, Error> { + ) -> Result, Error> { let mut request_spks = request.spks_by_keychain; // We keep track of already-scanned spks just in case a reorg happens and we need to do a @@ -134,20 +130,18 @@ impl ElectrumExt for E { }; }; - Ok(update) + Ok(ElectrumFullScanResult(update)) } - fn sync( - &self, - request: SyncRequest, - batch_size: usize, - ) -> Result, Error> { + fn sync(&self, request: SyncRequest, batch_size: usize) -> Result { let mut tx_cache = request.tx_cache.clone(); let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) .cache_txs(request.tx_cache) .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); - let mut full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size)?; + let mut full_scan_res = self + .full_scan(full_scan_req, usize::MAX, batch_size)? + .with_confirmation_height_anchor(); let (tip, _) = construct_update_tip(self, request.chain_tip)?; let cps = tip @@ -171,53 +165,64 @@ impl ElectrumExt for E { request.outpoints, )?; - Ok(SyncResult { + Ok(ElectrumSyncResult(SyncResult { chain_update: full_scan_res.chain_update, graph_update: full_scan_res.graph_update, - }) + })) } } -/// Trait that extends [`SyncResult`] and [`FullScanResult`] functionality. +/// The result of [`ElectrumExt::full_scan`]. /// -/// Currently, only a single method exists that converts the update [`TxGraph`] to have an anchor -/// type of [`ConfirmationTimeHeightAnchor`]. -pub trait ElectrumResultExt { - /// New result type with a [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`]. - type NewResult; - - /// Convert result type to have an update [`TxGraph`] that contains the [`ConfirmationTimeHeightAnchor`] . - fn try_into_confirmation_time_result( - self, - client: &impl ElectrumApi, - ) -> Result; -} - -impl ElectrumResultExt for FullScanResult { - type NewResult = FullScanResult; +/// This can be transformed into a [`FullScanResult`] with either [`ConfirmationHeightAnchor`] or +/// [`ConfirmationTimeHeightAnchor`] anchor types. +pub struct ElectrumFullScanResult(FullScanResult); + +impl ElectrumFullScanResult { + /// Return [`FullScanResult`] with [`ConfirmationHeightAnchor`]. + pub fn with_confirmation_height_anchor(self) -> FullScanResult { + self.0 + } - fn try_into_confirmation_time_result( + /// Return [`FullScanResult`] with [`ConfirmationTimeHeightAnchor`]. + /// + /// This requires additional calls to the Electrum server. + pub fn with_confirmation_time_height_anchor( self, client: &impl ElectrumApi, - ) -> Result { - Ok(FullScanResult:: { - graph_update: try_into_confirmation_time_result(self.graph_update, client)?, - chain_update: self.chain_update, - last_active_indices: self.last_active_indices, + ) -> Result, Error> { + let res = self.0; + Ok(FullScanResult { + graph_update: try_into_confirmation_time_result(res.graph_update, client)?, + chain_update: res.chain_update, + last_active_indices: res.last_active_indices, }) } } -impl ElectrumResultExt for SyncResult { - type NewResult = SyncResult; +/// The result of [`ElectrumExt::sync`]. +/// +/// This can be transformed into a [`SyncResult`] with either [`ConfirmationHeightAnchor`] or +/// [`ConfirmationTimeHeightAnchor`] anchor types. +pub struct ElectrumSyncResult(SyncResult); + +impl ElectrumSyncResult { + /// Return [`SyncResult`] with [`ConfirmationHeightAnchor`]. + pub fn with_confirmation_height_anchor(self) -> SyncResult { + self.0 + } - fn try_into_confirmation_time_result( + /// Return [`SyncResult`] with [`ConfirmationTimeHeightAnchor`]. + /// + /// This requires additional calls to the Electrum server. + pub fn with_confirmation_time_height_anchor( self, client: &impl ElectrumApi, - ) -> Result { + ) -> Result, Error> { + let res = self.0; Ok(SyncResult { - graph_update: try_into_confirmation_time_result(self.graph_update, client)?, - chain_update: self.chain_update, + graph_update: try_into_confirmation_time_result(res.graph_update, client)?, + chain_update: res.chain_update, }) } } diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index aa4b87933..9905ab9cc 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -5,7 +5,7 @@ use bdk_chain::{ spk_client::SyncRequest, ConfirmationTimeHeightAnchor, IndexedTxGraph, SpkTxOutIndex, }; -use bdk_electrum::{ElectrumExt, ElectrumResultExt}; +use bdk_electrum::ElectrumExt; use bdk_testenv::{anyhow, anyhow::Result, bitcoincore_rpc::RpcApi, TestEnv}; fn get_balance( @@ -67,7 +67,7 @@ fn scan_detects_confirmed_tx() -> Result<()> { .chain_spks(core::iter::once(spk_to_track)), 5, )? - .try_into_confirmation_time_result(&client)?; + .with_confirmation_time_height_anchor(&client)?; let _ = recv_chain .apply_update(update.chain_update) @@ -133,7 +133,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), 5, )? - .try_into_confirmation_time_result(&client)?; + .with_confirmation_time_height_anchor(&client)?; let _ = recv_chain .apply_update(update.chain_update) @@ -163,7 +163,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), 5, )? - .try_into_confirmation_time_result(&client)?; + .with_confirmation_time_height_anchor(&client)?; let _ = recv_chain .apply_update(update.chain_update) diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 58474668f..237d140a1 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -191,7 +191,8 @@ fn main() -> anyhow::Result<()> { let res = client .full_scan::<_>(request, stop_gap, scan_options.batch_size) - .context("scanning the blockchain")?; + .context("scanning the blockchain")? + .with_confirmation_height_anchor(); ( res.chain_update, res.graph_update, @@ -311,7 +312,8 @@ fn main() -> anyhow::Result<()> { let res = client .sync(request, scan_options.batch_size) - .context("scanning the blockchain")?; + .context("scanning the blockchain")? + .with_confirmation_height_anchor(); // drop lock on graph and chain drop((graph, chain)); diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index e1fe00984..a9b194ce8 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -10,7 +10,6 @@ use bdk::bitcoin::{Address, Amount}; use bdk::chain::collections::HashSet; use bdk::{bitcoin::Network, Wallet}; use bdk::{KeychainKind, SignOptions}; -use bdk_electrum::ElectrumResultExt; use bdk_electrum::{ electrum_client::{self, ElectrumApi}, ElectrumExt, @@ -55,7 +54,7 @@ fn main() -> Result<(), anyhow::Error> { let mut update = client .full_scan(request, STOP_GAP, BATCH_SIZE)? - .try_into_confirmation_time_result(&client)?; + .with_confirmation_time_height_anchor(&client)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); let _ = update.graph_update.update_last_seen_unconfirmed(now); From b2f3cacce6081f3bf6e103d1d2ca0707c545a67e Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Tue, 7 May 2024 19:57:11 +0800 Subject: [PATCH 11/13] feat(electrum): include option for previous `TxOut`s for fee calculation The previous `TxOut` for transactions received from an external wallet may be optionally added as floating `TxOut`s to `TxGraph` to allow for fee calculation. --- crates/electrum/src/electrum_ext.rs | 78 ++++++++++++++++----- crates/electrum/tests/test_electrum.rs | 26 +++++++ example-crates/example_electrum/src/main.rs | 4 +- example-crates/wallet_electrum/src/main.rs | 2 +- 4 files changed, 91 insertions(+), 19 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 806cabeb2..8559f5037 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -20,15 +20,18 @@ pub trait ElectrumExt { /// /// - `request`: struct with data required to perform a spk-based blockchain client full scan, /// see [`FullScanRequest`] - /// - /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated - /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a - /// single batch request. + /// - `stop_gap`: the full scan for each keychain stops after a gap of script pubkeys with no + /// associated transactions + /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch + /// request + /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee + /// calculation fn full_scan( &self, request: FullScanRequest, stop_gap: usize, batch_size: usize, + fetch_prev_txouts: bool, ) -> Result, Error>; /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified @@ -36,15 +39,21 @@ pub trait ElectrumExt { /// /// - `request`: struct with data required to perform a spk-based blockchain client sync, /// see [`SyncRequest`] - /// - /// `batch_size` specifies the max number of script pubkeys to request for in a single batch - /// request. + /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch + /// request + /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee + /// calculation /// /// If the scripts to sync are unknown, such as when restoring or importing a keychain that /// may include scripts that have been used, use [`full_scan`] with the keychain. /// /// [`full_scan`]: ElectrumExt::full_scan - fn sync(&self, request: SyncRequest, batch_size: usize) -> Result; + fn sync( + &self, + request: SyncRequest, + batch_size: usize, + fetch_prev_txouts: bool, + ) -> Result; } impl ElectrumExt for E { @@ -53,6 +62,7 @@ impl ElectrumExt for E { mut request: FullScanRequest, stop_gap: usize, batch_size: usize, + fetch_prev_txouts: bool, ) -> Result, Error> { let mut request_spks = request.spks_by_keychain; @@ -110,6 +120,11 @@ impl ElectrumExt for E { continue; // reorg } + // Fetch previous `TxOut`s for fee calculation if flag is enabled. + if fetch_prev_txouts { + fetch_prev_txout(self, &mut request.tx_cache, &mut graph_update)?; + } + let chain_update = tip; let keychain_update = request_spks @@ -133,14 +148,19 @@ impl ElectrumExt for E { Ok(ElectrumFullScanResult(update)) } - fn sync(&self, request: SyncRequest, batch_size: usize) -> Result { + fn sync( + &self, + request: SyncRequest, + batch_size: usize, + fetch_prev_txouts: bool, + ) -> Result { let mut tx_cache = request.tx_cache.clone(); let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone()) .cache_txs(request.tx_cache) .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk))); let mut full_scan_res = self - .full_scan(full_scan_req, usize::MAX, batch_size)? + .full_scan(full_scan_req, usize::MAX, batch_size, false)? .with_confirmation_height_anchor(); let (tip, _) = construct_update_tip(self, request.chain_tip)?; @@ -165,6 +185,11 @@ impl ElectrumExt for E { request.outpoints, )?; + // Fetch previous `TxOut`s for fee calculation if flag is enabled. + if fetch_prev_txouts { + fetch_prev_txout(self, &mut tx_cache, &mut full_scan_res.graph_update)?; + } + Ok(ElectrumSyncResult(SyncResult { chain_update: full_scan_res.chain_update, graph_update: full_scan_res.graph_update, @@ -374,7 +399,7 @@ fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, tx_cache: &mut TxCache, - tx_graph: &mut TxGraph, + graph_update: &mut TxGraph, outpoints: impl IntoIterator, ) -> Result<(), Error> { for outpoint in outpoints { @@ -399,18 +424,18 @@ fn populate_with_outpoints( continue; } has_residing = true; - if tx_graph.get_tx(res.tx_hash).is_none() { - let _ = tx_graph.insert_tx(tx.clone()); + if graph_update.get_tx(res.tx_hash).is_none() { + let _ = graph_update.insert_tx(tx.clone()); } } else { if has_spending { continue; } - let res_tx = match tx_graph.get_tx(res.tx_hash) { + let res_tx = match graph_update.get_tx(res.tx_hash) { Some(tx) => tx, None => { let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; - let _ = tx_graph.insert_tx(Arc::clone(&res_tx)); + let _ = graph_update.insert_tx(Arc::clone(&res_tx)); res_tx } }; @@ -424,7 +449,7 @@ fn populate_with_outpoints( }; if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { - let _ = tx_graph.insert_anchor(res.tx_hash, anchor); + let _ = graph_update.insert_anchor(res.tx_hash, anchor); } } } @@ -484,6 +509,27 @@ fn fetch_tx( }) } +// Helper function which fetches the `TxOut`s of our relevant transactions' previous transactions, +// which we do not have by default. This data is needed to calculate the transaction fee. +fn fetch_prev_txout( + client: &C, + tx_cache: &mut TxCache, + graph_update: &mut TxGraph, +) -> Result<(), Error> { + let full_txs: Vec> = + graph_update.full_txs().map(|tx_node| tx_node.tx).collect(); + for tx in full_txs { + for vin in &tx.input { + let outpoint = vin.previous_output; + let prev_tx = fetch_tx(client, tx_cache, outpoint.txid)?; + for txout in prev_tx.output.clone() { + let _ = graph_update.insert_txout(outpoint, txout); + } + } + } + Ok(()) +} + fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 9905ab9cc..1077fb8d9 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -66,6 +66,7 @@ fn scan_detects_confirmed_tx() -> Result<()> { SyncRequest::from_chain_tip(recv_chain.tip()) .chain_spks(core::iter::once(spk_to_track)), 5, + true, )? .with_confirmation_time_height_anchor(&client)?; @@ -83,6 +84,29 @@ fn scan_detects_confirmed_tx() -> Result<()> { }, ); + for tx in recv_graph.graph().full_txs() { + // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the + // floating txouts available from the transaction's previous outputs. + let fee = recv_graph + .graph() + .calculate_fee(&tx.tx) + .expect("fee must exist"); + + // Retrieve the fee in the transaction data from `bitcoind`. + let tx_fee = env + .bitcoind + .client + .get_transaction(&tx.txid, None) + .expect("Tx must exist") + .fee + .expect("Fee must exist") + .abs() + .to_sat() as u64; + + // Check that the calculated fee matches the fee from the transaction data. + assert_eq!(fee, tx_fee); + } + Ok(()) } @@ -132,6 +156,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { .sync( SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), 5, + false, )? .with_confirmation_time_height_anchor(&client)?; @@ -162,6 +187,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { .sync( SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]), 5, + false, )? .with_confirmation_time_height_anchor(&client)?; diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 237d140a1..e88b1e6fc 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -190,7 +190,7 @@ fn main() -> anyhow::Result<()> { }; let res = client - .full_scan::<_>(request, stop_gap, scan_options.batch_size) + .full_scan::<_>(request, stop_gap, scan_options.batch_size, false) .context("scanning the blockchain")? .with_confirmation_height_anchor(); ( @@ -311,7 +311,7 @@ fn main() -> anyhow::Result<()> { }); let res = client - .sync(request, scan_options.batch_size) + .sync(request, scan_options.batch_size, false) .context("scanning the blockchain")? .with_confirmation_height_anchor(); diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index a9b194ce8..eca96f32a 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -53,7 +53,7 @@ fn main() -> Result<(), anyhow::Error> { .inspect_spks_for_all_keychains(|_, _, _| std::io::stdout().flush().expect("must flush")); let mut update = client - .full_scan(request, STOP_GAP, BATCH_SIZE)? + .full_scan(request, STOP_GAP, BATCH_SIZE, false)? .with_confirmation_time_height_anchor(&client)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); From 92fb6cb37387fb0b9fe5329e772f0d928a33e116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Thu, 9 May 2024 16:38:56 +0800 Subject: [PATCH 12/13] chore(electrum): do not use `anyhow::Result` directly --- crates/electrum/tests/test_electrum.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index 1077fb8d9..dd7ee6a92 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -6,12 +6,12 @@ use bdk_chain::{ ConfirmationTimeHeightAnchor, IndexedTxGraph, SpkTxOutIndex, }; use bdk_electrum::ElectrumExt; -use bdk_testenv::{anyhow, anyhow::Result, bitcoincore_rpc::RpcApi, TestEnv}; +use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv}; fn get_balance( recv_chain: &LocalChain, recv_graph: &IndexedTxGraph>, -) -> Result { +) -> anyhow::Result { let chain_tip = recv_chain.tip().block_id(); let outpoints = recv_graph.index.outpoints().clone(); let balance = recv_graph @@ -27,7 +27,7 @@ fn get_balance( /// 3. Mine extra block to confirm sent tx. /// 4. Check [`Balance`] to ensure tx is confirmed. #[test] -fn scan_detects_confirmed_tx() -> Result<()> { +fn scan_detects_confirmed_tx() -> anyhow::Result<()> { const SEND_AMOUNT: Amount = Amount::from_sat(10_000); let env = TestEnv::new()?; @@ -117,7 +117,7 @@ fn scan_detects_confirmed_tx() -> Result<()> { /// 3. Perform 8 separate reorgs on each block with a confirmed tx. /// 4. Check [`Balance`] after each reorg to ensure unconfirmed amount is correct. #[test] -fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { +fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> { const REORG_COUNT: usize = 8; const SEND_AMOUNT: Amount = Amount::from_sat(10_000); From b45897e6fe2f7e67f5d75ec6f983757b28c5ec19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BF=97=E5=AE=87?= Date: Fri, 10 May 2024 16:40:55 +0800 Subject: [PATCH 13/13] feat(electrum): update docs and simplify logic of `ElectrumExt` Helper method docs are updated to explain what they are updating. Logic is simplified as we do not need to check whether a tx exists already in `update_graph` before inserting it. --- crates/electrum/src/electrum_ext.rs | 69 ++++++++++++++++------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 8559f5037..143446951 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -395,6 +395,12 @@ fn determine_tx_anchor( } } +/// Populate the `graph_update` with associated transactions/anchors of `outpoints`. +/// +/// Transactions in which the outpoint resides, and transactions that spend from the outpoint are +/// included. Anchors of the aforementioned transactions are included. +/// +/// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, @@ -403,42 +409,34 @@ fn populate_with_outpoints( outpoints: impl IntoIterator, ) -> Result<(), Error> { for outpoint in outpoints { - let txid = outpoint.txid; - let tx = client.transaction_get(&txid)?; - debug_assert_eq!(tx.txid(), txid); - let txout = match tx.output.get(outpoint.vout as usize) { + let op_txid = outpoint.txid; + let op_tx = fetch_tx(client, tx_cache, op_txid)?; + let op_txout = match op_tx.output.get(outpoint.vout as usize) { Some(txout) => txout, None => continue, }; + debug_assert_eq!(op_tx.txid(), op_txid); + // attempt to find the following transactions (alongside their chain positions), and // add to our sparsechain `update`: let mut has_residing = false; // tx in which the outpoint resides let mut has_spending = false; // tx that spends the outpoint - for res in client.script_get_history(&txout.script_pubkey)? { + for res in client.script_get_history(&op_txout.script_pubkey)? { if has_residing && has_spending { break; } - if res.tx_hash == txid { - if has_residing { - continue; - } + if !has_residing && res.tx_hash == op_txid { has_residing = true; - if graph_update.get_tx(res.tx_hash).is_none() { - let _ = graph_update.insert_tx(tx.clone()); + let _ = graph_update.insert_tx(Arc::clone(&op_tx)); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = graph_update.insert_anchor(res.tx_hash, anchor); } - } else { - if has_spending { - continue; - } - let res_tx = match graph_update.get_tx(res.tx_hash) { - Some(tx) => tx, - None => { - let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; - let _ = graph_update.insert_tx(Arc::clone(&res_tx)); - res_tx - } - }; + } + + if !has_spending && res.tx_hash != op_txid { + let res_tx = fetch_tx(client, tx_cache, res.tx_hash)?; + // we exclude txs/anchors that do not spend our specified outpoint(s) has_spending = res_tx .input .iter() @@ -446,16 +444,17 @@ fn populate_with_outpoints( if !has_spending { continue; } - }; - - if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { - let _ = graph_update.insert_anchor(res.tx_hash, anchor); + let _ = graph_update.insert_tx(Arc::clone(&res_tx)); + if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) { + let _ = graph_update.insert_anchor(res.tx_hash, anchor); + } } } } Ok(()) } +/// Populate the `graph_update` with transactions/anchors of the provided `txids`. fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, @@ -476,6 +475,8 @@ fn populate_with_txids( .map(|txo| &txo.script_pubkey) .expect("tx must have an output"); + // because of restrictions of the Electrum API, we have to use the `script_get_history` + // call to get confirmation status of our transaction let anchor = match client .script_get_history(spk)? .into_iter() @@ -485,9 +486,7 @@ fn populate_with_txids( None => continue, }; - if graph_update.get_tx(txid).is_none() { - let _ = graph_update.insert_tx(tx); - } + let _ = graph_update.insert_tx(tx); if let Some(anchor) = anchor { let _ = graph_update.insert_anchor(txid, anchor); } @@ -495,6 +494,9 @@ fn populate_with_txids( Ok(()) } +/// Fetch transaction of given `txid`. +/// +/// We maintain a `tx_cache` so that we won't need to fetch from Electrum with every call. fn fetch_tx( client: &C, tx_cache: &mut TxCache, @@ -530,6 +532,13 @@ fn fetch_prev_txout( Ok(()) } +/// Populate the `graph_update` with transactions/anchors associated with the given `spks`. +/// +/// Transactions that contains an output with requested spk, or spends form an output with +/// requested spk will be added to `graph_update`. Anchors of the aforementioned transactions are +/// also included. +/// +/// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory. fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap,