Skip to content

Commit

Permalink
refactor(electrum): remove RelevantTxids and track txs in TxGraph
Browse files Browse the repository at this point in the history 
This PR removes `RelevantTxids` from the electrum crate and tracks
transactions in a `TxGraph`. This removes the need to separately
construct a `TxGraph` after a `full_scan` or `sync`.
  • Loading branch information
@LagginTimes
LagginTimes committed Apr 11, 2024
1 parent 62619d3 commit 53ed564
Show file tree
Hide file tree
Showing 4 changed files with 73 additions and 165 deletions.
207 changes: 62 additions & 145 deletions crates/electrum/src/electrum_ext.rs
View file
Original file line number Diff line number Diff line change
@@ -1,122 +1,15 @@
use bdk_chain::{
bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
bitcoin::{OutPoint, ScriptBuf, Txid},
local_chain::{self, CheckPoint},
tx_graph::{self, TxGraph},
Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor,
};
use electrum_client::{Client, ElectrumApi, Error, HeaderNotification};
use std::{
collections::{BTreeMap, BTreeSet, HashMap, HashSet},
fmt::Debug,
str::FromStr,
tx_graph::TxGraph,
BlockId, ConfirmationTimeHeightAnchor,
};
use electrum_client::{ElectrumApi, Error, HeaderNotification};
use std::{collections::BTreeMap, fmt::Debug, str::FromStr};

/// We include a chain suffix of a certain length for the purpose of robustness.
const CHAIN_SUFFIX_LENGTH: u32 = 8;

/// Represents updates fetched from an Electrum server, but excludes full transactions.
///
/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to
/// fetch the full transactions from Electrum and finalize the update.
#[derive(Debug, Default, Clone)]
pub struct RelevantTxids(HashMap<Txid, BTreeSet<ConfirmationHeightAnchor>>);

impl RelevantTxids {
/// Determine the full transactions that are missing from `graph`.
///
/// Refer to [`RelevantTxids`] for more details.
pub fn missing_full_txs<A: Anchor>(&self, graph: &TxGraph<A>) -> Vec<Txid> {
self.0
.keys()
.filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
.cloned()
.collect()
}

/// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`.
///
/// Refer to [`RelevantTxids`] for more details.
pub fn into_tx_graph(
self,
client: &Client,
missing: Vec<Txid>,
) -> Result<TxGraph<ConfirmationHeightAnchor>, Error> {
let new_txs = client.batch_transaction_get(&missing)?;
let mut graph = TxGraph::<ConfirmationHeightAnchor>::new(new_txs);
for (txid, anchors) in self.0 {
for anchor in anchors {
let _ = graph.insert_anchor(txid, anchor);
}
}
Ok(graph)
}

/// Finalizes the update by fetching `missing` txids from the `client`, where the
/// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`].
///
/// Refer to [`RelevantTxids`] for more details.
///
/// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
// use it.
pub fn into_confirmation_time_tx_graph(
self,
client: &Client,
missing: Vec<Txid>,
) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
let graph = self.into_tx_graph(client, missing)?;

let relevant_heights = {
let mut visited_heights = HashSet::new();
graph
.all_anchors()
.iter()
.map(|(a, _)| a.confirmation_height_upper_bound())
.filter(move |&h| visited_heights.insert(h))
.collect::<Vec<_>>()
};

let height_to_time = relevant_heights
.clone()
.into_iter()
.zip(
client
.batch_block_header(relevant_heights)?
.into_iter()
.map(|bh| bh.time as u64),
)
.collect::<HashMap<u32, u64>>();

let graph_changeset = {
let old_changeset = TxGraph::default().apply_update(graph);
tx_graph::ChangeSet {
txs: old_changeset.txs,
txouts: old_changeset.txouts,
last_seen: old_changeset.last_seen,
anchors: old_changeset
.anchors
.into_iter()
.map(|(height_anchor, txid)| {
let confirmation_height = height_anchor.confirmation_height;
let confirmation_time = height_to_time[&confirmation_height];
let time_anchor = ConfirmationTimeHeightAnchor {
anchor_block: height_anchor.anchor_block,
confirmation_height,
confirmation_time,
};
(time_anchor, txid)
})
.collect(),
}
};

let mut new_graph = TxGraph::default();
new_graph.apply_changeset(graph_changeset);
Ok(new_graph)
}
}

/// Combination of chain and transactions updates from electrum
///
/// We have to update the chain and the txids at the same time since we anchor the txids to
Expand All @@ -125,8 +18,8 @@ impl RelevantTxids {
pub struct ElectrumUpdate {
/// Chain update
pub chain_update: local_chain::Update,
/// Transaction updates from electrum
pub relevant_txids: RelevantTxids,
/// Tracks electrum updates in TxGraph
pub graph_update: TxGraph<ConfirmationTimeHeightAnchor>,
}

/// Trait to extend [`Client`] functionality.
Expand Down Expand Up @@ -190,7 +83,7 @@ impl<A: ElectrumApi> ElectrumExt for A {

let (electrum_update, keychain_update) = loop {
let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
let mut relevant_txids = RelevantTxids::default();
let mut graph_update = TxGraph::<ConfirmationTimeHeightAnchor>::default();
let cps = tip
.iter()
.take(10)
Expand All @@ -202,7 +95,7 @@ impl<A: ElectrumApi> ElectrumExt for A {
scanned_spks.append(&mut populate_with_spks(
self,
&cps,
&mut relevant_txids,
&mut graph_update,
&mut scanned_spks
.iter()
.map(|(i, (spk, _))| (i.clone(), spk.clone())),
Expand All @@ -215,7 +108,7 @@ impl<A: ElectrumApi> ElectrumExt for A {
populate_with_spks(
self,
&cps,
&mut relevant_txids,
&mut graph_update,
keychain_spks,
stop_gap,
batch_size,
Expand Down Expand Up @@ -251,7 +144,7 @@ impl<A: ElectrumApi> ElectrumExt for A {
break (
ElectrumUpdate {
chain_update,
relevant_txids,
graph_update,
},
keychain_update,
);
Expand Down Expand Up @@ -287,10 +180,8 @@ impl<A: ElectrumApi> ElectrumExt for A {
.map(|cp| (cp.height(), cp))
.collect::<BTreeMap<u32, CheckPoint>>();

populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?;

let _txs =
populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?;
populate_with_txids(self, &cps, &mut electrum_update.graph_update, txids)?;
populate_with_outpoints(self, &cps, &mut electrum_update.graph_update, outpoints)?;

Ok(electrum_update)
}
Expand Down Expand Up @@ -373,10 +264,16 @@ fn construct_update_tip(
///
/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
fn determine_tx_anchor(
client: &impl ElectrumApi,
cps: &BTreeMap<u32, CheckPoint>,
raw_height: i32,
txid: Txid,
) -> Option<ConfirmationHeightAnchor> {
) -> Option<ConfirmationTimeHeightAnchor> {
let confirmation_time = client
.block_header(raw_height as usize)
.expect("header must exist")
.time as u64;

// The electrum API has a weird quirk where an unconfirmed transaction is presented with a
// height of 0. To avoid invalid representation in our data structures, we manually set
// transactions residing in the genesis block to have height 0, then interpret a height of 0 as
Expand All @@ -386,9 +283,10 @@ fn determine_tx_anchor(
.expect("must deserialize genesis coinbase txid")
{
let anchor_block = cps.values().next()?.block_id();
return Some(ConfirmationHeightAnchor {
return Some(ConfirmationTimeHeightAnchor {
anchor_block,
confirmation_height: 0,
confirmation_time,
});
}
match raw_height {
Expand All @@ -402,9 +300,10 @@ fn determine_tx_anchor(
if h > anchor_block.height {
None
} else {
Some(ConfirmationHeightAnchor {
Some(ConfirmationTimeHeightAnchor {
anchor_block,
confirmation_height: h,
confirmation_time,
})
}
}
Expand All @@ -414,10 +313,9 @@ fn determine_tx_anchor(
fn populate_with_outpoints(
client: &impl ElectrumApi,
cps: &BTreeMap<u32, CheckPoint>,
relevant_txids: &mut RelevantTxids,
tx_graph: &mut TxGraph<ConfirmationTimeHeightAnchor>,
outpoints: impl IntoIterator<Item = OutPoint>,
) -> Result<HashMap<Txid, Transaction>, Error> {
let mut full_txs = HashMap::new();
) -> Result<(), Error> {
for outpoint in outpoints {
let txid = outpoint.txid;
let tx = client.transaction_get(&txid)?;
Expand All @@ -431,6 +329,8 @@ fn populate_with_outpoints(
let mut has_residing = false; // tx in which the outpoint resides
let mut has_spending = false; // tx that spends the outpoint
for res in client.script_get_history(&txout.script_pubkey)? {
let mut update = TxGraph::<ConfirmationTimeHeightAnchor>::default();

if has_residing && has_spending {
break;
}
Expand All @@ -440,17 +340,19 @@ fn populate_with_outpoints(
continue;
}
has_residing = true;
full_txs.insert(res.tx_hash, tx.clone());
if tx_graph.get_tx(res.tx_hash).is_none() {
update = TxGraph::<ConfirmationTimeHeightAnchor>::new([tx.clone()]);
}
} else {
if has_spending {
continue;
}
let res_tx = match full_txs.get(&res.tx_hash) {
let res_tx = match tx_graph.get_tx(res.tx_hash) {
Some(tx) => tx,
None => {
let res_tx = client.transaction_get(&res.tx_hash)?;
full_txs.insert(res.tx_hash, res_tx);
full_txs.get(&res.tx_hash).expect("just inserted")
update = TxGraph::<ConfirmationTimeHeightAnchor>::new([res_tx]);
tx_graph.get_tx(res.tx_hash).expect("just inserted")
}
};
has_spending = res_tx
Expand All @@ -462,23 +364,25 @@ fn populate_with_outpoints(
}
};

let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);
let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default();
if let Some(anchor) = anchor {
tx_entry.insert(anchor);
if let Some(anchor) = determine_tx_anchor(client, cps, res.height, res.tx_hash) {
let _ = update.insert_anchor(res.tx_hash, anchor);
}

let _ = tx_graph.apply_update(update);
}
}
Ok(full_txs)
Ok(())
}

fn populate_with_txids(
client: &impl ElectrumApi,
cps: &BTreeMap<u32, CheckPoint>,
relevant_txids: &mut RelevantTxids,
tx_graph: &mut TxGraph<ConfirmationTimeHeightAnchor>,
txids: impl IntoIterator<Item = Txid>,
) -> Result<(), Error> {
for txid in txids {
let mut update = TxGraph::<ConfirmationTimeHeightAnchor>::default();

let tx = match client.transaction_get(&txid) {
Ok(tx) => tx,
Err(electrum_client::Error::Protocol(_)) => continue,
Expand All @@ -496,22 +400,27 @@ fn populate_with_txids(
.into_iter()
.find(|r| r.tx_hash == txid)
{
Some(r) => determine_tx_anchor(cps, r.height, txid),
Some(r) => determine_tx_anchor(client, cps, r.height, txid),
None => continue,
};

let tx_entry = relevant_txids.0.entry(txid).or_default();
if tx_graph.get_tx(txid).is_none() {
update = TxGraph::<ConfirmationTimeHeightAnchor>::new([tx]);
}

if let Some(anchor) = anchor {
tx_entry.insert(anchor);
let _ = update.insert_anchor(txid, anchor);
}

let _ = tx_graph.apply_update(update);
}
Ok(())
}

fn populate_with_spks<I: Ord + Clone>(
client: &impl ElectrumApi,
cps: &BTreeMap<u32, CheckPoint>,
relevant_txids: &mut RelevantTxids,
tx_graph: &mut TxGraph<ConfirmationTimeHeightAnchor>,
spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
stop_gap: usize,
batch_size: usize,
Expand Down Expand Up @@ -544,10 +453,18 @@ fn populate_with_spks<I: Ord + Clone>(
}

for tx in spk_history {
let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default();
if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {
tx_entry.insert(anchor);
let mut update = TxGraph::<ConfirmationTimeHeightAnchor>::default();

if tx_graph.get_tx(tx.tx_hash).is_none() {
let full_tx = client.transaction_get(&tx.tx_hash)?;
update = TxGraph::<ConfirmationTimeHeightAnchor>::new([full_tx]);
}

if let Some(anchor) = determine_tx_anchor(client, cps, tx.height, tx.tx_hash) {
let _ = update.insert_anchor(tx.tx_hash, anchor);
}

let _ = tx_graph.apply_update(update);
}
}
}
Expand Down
Loading

0 comments on commit 53ed564

Please sign in to comment.