From 53ed564435dc789893102c50e1d516e4b6df14e8 Mon Sep 17 00:00:00 2001 From: Wei Chen Date: Thu, 11 Apr 2024 17:57:14 -0400 Subject: [PATCH] refactor(electrum): remove `RelevantTxids` and track txs in `TxGraph` This PR removes `RelevantTxids` from the electrum crate and tracks transactions in a `TxGraph`. This removes the need to separately construct a `TxGraph` after a `full_scan` or `sync`. --- crates/electrum/src/electrum_ext.rs | 207 ++++++-------------- crates/electrum/tests/test_electrum.rs | 12 +- example-crates/example_electrum/src/main.rs | 15 +- example-crates/wallet_electrum/src/main.rs | 4 +- 4 files changed, 73 insertions(+), 165 deletions(-) diff --git a/crates/electrum/src/electrum_ext.rs b/crates/electrum/src/electrum_ext.rs index 3ff467fcee..9fa932ff7b 100644 --- a/crates/electrum/src/electrum_ext.rs +++ b/crates/electrum/src/electrum_ext.rs @@ -1,122 +1,15 @@ use bdk_chain::{ - bitcoin::{OutPoint, ScriptBuf, Transaction, Txid}, + bitcoin::{OutPoint, ScriptBuf, Txid}, local_chain::{self, CheckPoint}, - tx_graph::{self, TxGraph}, - Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor, -}; -use electrum_client::{Client, ElectrumApi, Error, HeaderNotification}; -use std::{ - collections::{BTreeMap, BTreeSet, HashMap, HashSet}, - fmt::Debug, - str::FromStr, + tx_graph::TxGraph, + BlockId, ConfirmationTimeHeightAnchor, }; +use electrum_client::{ElectrumApi, Error, HeaderNotification}; +use std::{collections::BTreeMap, fmt::Debug, str::FromStr}; /// We include a chain suffix of a certain length for the purpose of robustness. const CHAIN_SUFFIX_LENGTH: u32 = 8; -/// Represents updates fetched from an Electrum server, but excludes full transactions. -/// -/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to -/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to -/// fetch the full transactions from Electrum and finalize the update. -#[derive(Debug, Default, Clone)] -pub struct RelevantTxids(HashMap>); - -impl RelevantTxids { - /// Determine the full transactions that are missing from `graph`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn missing_full_txs(&self, graph: &TxGraph) -> Vec { - self.0 - .keys() - .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none()) - .cloned() - .collect() - } - - /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`. - /// - /// Refer to [`RelevantTxids`] for more details. - pub fn into_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let new_txs = client.batch_transaction_get(&missing)?; - let mut graph = TxGraph::::new(new_txs); - for (txid, anchors) in self.0 { - for anchor in anchors { - let _ = graph.insert_anchor(txid, anchor); - } - } - Ok(graph) - } - - /// Finalizes the update by fetching `missing` txids from the `client`, where the - /// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`]. - /// - /// Refer to [`RelevantTxids`] for more details. - /// - /// **Note:** The confirmation time might not be precisely correct if there has been a reorg. - // Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to - // use it. - pub fn into_confirmation_time_tx_graph( - self, - client: &Client, - missing: Vec, - ) -> Result, Error> { - let graph = self.into_tx_graph(client, missing)?; - - let relevant_heights = { - let mut visited_heights = HashSet::new(); - graph - .all_anchors() - .iter() - .map(|(a, _)| a.confirmation_height_upper_bound()) - .filter(move |&h| visited_heights.insert(h)) - .collect::>() - }; - - let height_to_time = relevant_heights - .clone() - .into_iter() - .zip( - client - .batch_block_header(relevant_heights)? - .into_iter() - .map(|bh| bh.time as u64), - ) - .collect::>(); - - let graph_changeset = { - let old_changeset = TxGraph::default().apply_update(graph); - tx_graph::ChangeSet { - txs: old_changeset.txs, - txouts: old_changeset.txouts, - last_seen: old_changeset.last_seen, - anchors: old_changeset - .anchors - .into_iter() - .map(|(height_anchor, txid)| { - let confirmation_height = height_anchor.confirmation_height; - let confirmation_time = height_to_time[&confirmation_height]; - let time_anchor = ConfirmationTimeHeightAnchor { - anchor_block: height_anchor.anchor_block, - confirmation_height, - confirmation_time, - }; - (time_anchor, txid) - }) - .collect(), - } - }; - - let mut new_graph = TxGraph::default(); - new_graph.apply_changeset(graph_changeset); - Ok(new_graph) - } -} - /// Combination of chain and transactions updates from electrum /// /// We have to update the chain and the txids at the same time since we anchor the txids to @@ -125,8 +18,8 @@ impl RelevantTxids { pub struct ElectrumUpdate { /// Chain update pub chain_update: local_chain::Update, - /// Transaction updates from electrum - pub relevant_txids: RelevantTxids, + /// Tracks electrum updates in TxGraph + pub graph_update: TxGraph, } /// Trait to extend [`Client`] functionality. @@ -190,7 +83,7 @@ impl ElectrumExt for A { let (electrum_update, keychain_update) = loop { let (tip, _) = construct_update_tip(self, prev_tip.clone())?; - let mut relevant_txids = RelevantTxids::default(); + let mut graph_update = TxGraph::::default(); let cps = tip .iter() .take(10) @@ -202,7 +95,7 @@ impl ElectrumExt for A { scanned_spks.append(&mut populate_with_spks( self, &cps, - &mut relevant_txids, + &mut graph_update, &mut scanned_spks .iter() .map(|(i, (spk, _))| (i.clone(), spk.clone())), @@ -215,7 +108,7 @@ impl ElectrumExt for A { populate_with_spks( self, &cps, - &mut relevant_txids, + &mut graph_update, keychain_spks, stop_gap, batch_size, @@ -251,7 +144,7 @@ impl ElectrumExt for A { break ( ElectrumUpdate { chain_update, - relevant_txids, + graph_update, }, keychain_update, ); @@ -287,10 +180,8 @@ impl ElectrumExt for A { .map(|cp| (cp.height(), cp)) .collect::>(); - populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?; - - let _txs = - populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?; + populate_with_txids(self, &cps, &mut electrum_update.graph_update, txids)?; + populate_with_outpoints(self, &cps, &mut electrum_update.graph_update, outpoints)?; Ok(electrum_update) } @@ -373,10 +264,16 @@ fn construct_update_tip( /// /// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status) fn determine_tx_anchor( + client: &impl ElectrumApi, cps: &BTreeMap, raw_height: i32, txid: Txid, -) -> Option { +) -> Option { + let confirmation_time = client + .block_header(raw_height as usize) + .expect("header must exist") + .time as u64; + // The electrum API has a weird quirk where an unconfirmed transaction is presented with a // height of 0. To avoid invalid representation in our data structures, we manually set // transactions residing in the genesis block to have height 0, then interpret a height of 0 as @@ -386,9 +283,10 @@ fn determine_tx_anchor( .expect("must deserialize genesis coinbase txid") { let anchor_block = cps.values().next()?.block_id(); - return Some(ConfirmationHeightAnchor { + return Some(ConfirmationTimeHeightAnchor { anchor_block, confirmation_height: 0, + confirmation_time, }); } match raw_height { @@ -402,9 +300,10 @@ fn determine_tx_anchor( if h > anchor_block.height { None } else { - Some(ConfirmationHeightAnchor { + Some(ConfirmationTimeHeightAnchor { anchor_block, confirmation_height: h, + confirmation_time, }) } } @@ -414,10 +313,9 @@ fn determine_tx_anchor( fn populate_with_outpoints( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, outpoints: impl IntoIterator, -) -> Result, Error> { - let mut full_txs = HashMap::new(); +) -> Result<(), Error> { for outpoint in outpoints { let txid = outpoint.txid; let tx = client.transaction_get(&txid)?; @@ -431,6 +329,8 @@ fn populate_with_outpoints( let mut has_residing = false; // tx in which the outpoint resides let mut has_spending = false; // tx that spends the outpoint for res in client.script_get_history(&txout.script_pubkey)? { + let mut update = TxGraph::::default(); + if has_residing && has_spending { break; } @@ -440,17 +340,19 @@ fn populate_with_outpoints( continue; } has_residing = true; - full_txs.insert(res.tx_hash, tx.clone()); + if tx_graph.get_tx(res.tx_hash).is_none() { + update = TxGraph::::new([tx.clone()]); + } } else { if has_spending { continue; } - let res_tx = match full_txs.get(&res.tx_hash) { + let res_tx = match tx_graph.get_tx(res.tx_hash) { Some(tx) => tx, None => { let res_tx = client.transaction_get(&res.tx_hash)?; - full_txs.insert(res.tx_hash, res_tx); - full_txs.get(&res.tx_hash).expect("just inserted") + update = TxGraph::::new([res_tx]); + tx_graph.get_tx(res.tx_hash).expect("just inserted") } }; has_spending = res_tx @@ -462,23 +364,25 @@ fn populate_with_outpoints( } }; - let anchor = determine_tx_anchor(cps, res.height, res.tx_hash); - let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default(); - if let Some(anchor) = anchor { - tx_entry.insert(anchor); + if let Some(anchor) = determine_tx_anchor(client, cps, res.height, res.tx_hash) { + let _ = update.insert_anchor(res.tx_hash, anchor); } + + let _ = tx_graph.apply_update(update); } } - Ok(full_txs) + Ok(()) } fn populate_with_txids( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, txids: impl IntoIterator, ) -> Result<(), Error> { for txid in txids { + let mut update = TxGraph::::default(); + let tx = match client.transaction_get(&txid) { Ok(tx) => tx, Err(electrum_client::Error::Protocol(_)) => continue, @@ -496,14 +400,19 @@ fn populate_with_txids( .into_iter() .find(|r| r.tx_hash == txid) { - Some(r) => determine_tx_anchor(cps, r.height, txid), + Some(r) => determine_tx_anchor(client, cps, r.height, txid), None => continue, }; - let tx_entry = relevant_txids.0.entry(txid).or_default(); + if tx_graph.get_tx(txid).is_none() { + update = TxGraph::::new([tx]); + } + if let Some(anchor) = anchor { - tx_entry.insert(anchor); + let _ = update.insert_anchor(txid, anchor); } + + let _ = tx_graph.apply_update(update); } Ok(()) } @@ -511,7 +420,7 @@ fn populate_with_txids( fn populate_with_spks( client: &impl ElectrumApi, cps: &BTreeMap, - relevant_txids: &mut RelevantTxids, + tx_graph: &mut TxGraph, spks: &mut impl Iterator, stop_gap: usize, batch_size: usize, @@ -544,10 +453,18 @@ fn populate_with_spks( } for tx in spk_history { - let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default(); - if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) { - tx_entry.insert(anchor); + let mut update = TxGraph::::default(); + + if tx_graph.get_tx(tx.tx_hash).is_none() { + let full_tx = client.transaction_get(&tx.tx_hash)?; + update = TxGraph::::new([full_tx]); + } + + if let Some(anchor) = determine_tx_anchor(client, cps, tx.height, tx.tx_hash) { + let _ = update.insert_anchor(tx.tx_hash, anchor); } + + let _ = tx_graph.apply_update(update); } } } diff --git a/crates/electrum/tests/test_electrum.rs b/crates/electrum/tests/test_electrum.rs index e6c93651d8..f915fc081e 100644 --- a/crates/electrum/tests/test_electrum.rs +++ b/crates/electrum/tests/test_electrum.rs @@ -64,11 +64,9 @@ fn scan_detects_confirmed_tx() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, + graph_update, } = client.sync(recv_chain.tip(), [spk_to_track], None, None, 5)?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -130,11 +128,9 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, + graph_update, } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; @@ -160,11 +156,9 @@ fn tx_can_become_unconfirmed_after_reorg() -> Result<()> { env.wait_until_electrum_sees_block()?; let ElectrumUpdate { chain_update, - relevant_txids, + graph_update, } = client.sync(recv_chain.tip(), [spk_to_track.clone()], None, None, 5)?; - let missing = relevant_txids.missing_full_txs(recv_graph.graph()); - let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let _ = recv_chain .apply_update(chain_update) .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?; diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index f651b85e27..b05549d3a9 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -289,17 +289,11 @@ fn main() -> anyhow::Result<()> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, ) = response; - let missing_txids = { - let graph = &*graph.lock().unwrap(); - relevant_txids.missing_full_txs(graph.graph()) - }; - - let mut graph_update = relevant_txids.into_tx_graph(&client, missing_txids)?; let now = std::time::UNIX_EPOCH .elapsed() .expect("must get time") @@ -320,7 +314,12 @@ fn main() -> anyhow::Result<()> { indexer, ..Default::default() }); - changeset.append(graph.apply_update(graph_update)); + changeset.append(graph.apply_update(graph_update.map_anchors(|a| { + ConfirmationHeightAnchor { + anchor_block: a.anchor_block, + confirmation_height: a.confirmation_height, + } + }))); changeset }; diff --git a/example-crates/wallet_electrum/src/main.rs b/example-crates/wallet_electrum/src/main.rs index e53e19d4ff..5d80b27703 100644 --- a/example-crates/wallet_electrum/src/main.rs +++ b/example-crates/wallet_electrum/src/main.rs @@ -58,15 +58,13 @@ fn main() -> Result<(), anyhow::Error> { let ( ElectrumUpdate { chain_update, - relevant_txids, + mut graph_update, }, keychain_update, ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?; println!(); - let missing = relevant_txids.missing_full_txs(wallet.as_ref()); - let mut graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, missing)?; let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs(); let _ = graph_update.update_last_seen_unconfirmed(now);