diff --git a/Cargo.toml b/Cargo.toml index 9fafb8b78..e818d8996 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,6 +7,7 @@ members = [ "crates/esplora", "example-crates/example_cli", "example-crates/example_electrum", + "example-crates/example_esplora", "example-crates/wallet_electrum", "example-crates/wallet_esplora_blocking", "example-crates/wallet_esplora_async", diff --git a/crates/bdk/src/wallet/mod.rs b/crates/bdk/src/wallet/mod.rs index 1ca78a775..d9ba3f1ee 100644 --- a/crates/bdk/src/wallet/mod.rs +++ b/crates/bdk/src/wallet/mod.rs @@ -247,7 +247,7 @@ impl Wallet { let changeset = db.load_from_persistence().map_err(NewError::Persist)?; chain.apply_changeset(&changeset.chain); - indexed_graph.apply_changeset(changeset.index_tx_graph); + indexed_graph.apply_changeset(changeset.indexed_tx_graph); let persist = Persist::new(db); diff --git a/crates/chain/src/keychain.rs b/crates/chain/src/keychain.rs index 64d68d81e..3a6cddf80 100644 --- a/crates/chain/src/keychain.rs +++ b/crates/chain/src/keychain.rs @@ -133,14 +133,14 @@ pub struct WalletChangeSet { /// ChangeSet to [`IndexedTxGraph`]. /// /// [`IndexedTxGraph`]: crate::indexed_tx_graph::IndexedTxGraph - pub index_tx_graph: indexed_tx_graph::ChangeSet>, + pub indexed_tx_graph: indexed_tx_graph::ChangeSet>, } impl Default for WalletChangeSet { fn default() -> Self { Self { chain: Default::default(), - index_tx_graph: Default::default(), + indexed_tx_graph: Default::default(), } } } @@ -148,11 +148,11 @@ impl Default for WalletChangeSet { impl Append for WalletChangeSet { fn append(&mut self, other: Self) { Append::append(&mut self.chain, other.chain); - Append::append(&mut self.index_tx_graph, other.index_tx_graph); + Append::append(&mut self.indexed_tx_graph, other.indexed_tx_graph); } fn is_empty(&self) -> bool { - self.chain.is_empty() && self.index_tx_graph.is_empty() + self.chain.is_empty() && self.indexed_tx_graph.is_empty() } } @@ -166,9 +166,9 @@ impl From for WalletChangeSet { } impl From>> for WalletChangeSet { - fn from(index_tx_graph: indexed_tx_graph::ChangeSet>) -> Self { + fn from(indexed_tx_graph: indexed_tx_graph::ChangeSet>) -> Self { Self { - index_tx_graph, + indexed_tx_graph, ..Default::default() } } diff --git a/crates/chain/src/keychain/txout_index.rs b/crates/chain/src/keychain/txout_index.rs index 9b38a7ade..70a475c29 100644 --- a/crates/chain/src/keychain/txout_index.rs +++ b/crates/chain/src/keychain/txout_index.rs @@ -270,12 +270,7 @@ impl KeychainTxOutIndex { ) -> BTreeMap>> { self.keychains .iter() - .map(|(keychain, descriptor)| { - ( - keychain.clone(), - SpkIterator::new_with_range(descriptor.clone(), 0..), - ) - }) + .map(|(keychain, descriptor)| (keychain.clone(), SpkIterator::new(descriptor.clone()))) .collect() } @@ -291,7 +286,7 @@ impl KeychainTxOutIndex { .get(keychain) .expect("keychain must exist") .clone(); - SpkIterator::new_with_range(descriptor, 0..) + SpkIterator::new(descriptor) } /// Convenience method to get [`revealed_spks_of_keychain`] of all keychains. diff --git a/crates/chain/src/tx_graph.rs b/crates/chain/src/tx_graph.rs index 5912921a9..da0e4eca1 100644 --- a/crates/chain/src/tx_graph.rs +++ b/crates/chain/src/tx_graph.rs @@ -1048,6 +1048,33 @@ impl ChangeSet { }) .chain(self.txouts.iter().map(|(op, txout)| (*op, txout))) } + + /// Iterates over the heights of that the new transaction anchors in this changeset. + /// + /// This is useful if you want to find which heights you need to fetch data about in order to + /// confirm or exclude these anchors. + /// + /// See also: [`Self::missing_heights`] + pub fn relevant_heights(&self) -> impl Iterator + '_ where A: Anchor { + let mut dedup = None; + self.anchors.iter().map(|(a, _)| a.anchor_block().height).filter(move |height| { + let duplicate = dedup == Some(*height); + dedup = Some(*height); + !duplicate + }) + } + + + /// Returns an iterator for the [`relevant_heights`] in this changeset that are not included in + /// `local_chain`. This tells you which heights you need to include in `local_chain` in order + /// for it to conclusively act as a [`ChainOracle`] for the transaction anchors this changeset + /// will add. + /// + /// [`ChainOracle`]: crate::ChainOracle + /// [`relevant_heights`]: Self::relevant_heights + pub fn missing_heights_from<'a>(&'a self, local_chain: &'a LocalChain) -> impl Iterator + 'a where A: Anchor { + self.relevant_heights().filter(move |height| !local_chain.blocks().contains_key(height)) + } } impl Append for ChangeSet { diff --git a/example-crates/example_electrum/src/main.rs b/example-crates/example_electrum/src/main.rs index 2a5c1310c..4c5fde6a3 100644 --- a/example-crates/example_electrum/src/main.rs +++ b/example-crates/example_electrum/src/main.rs @@ -68,7 +68,7 @@ fn main() -> anyhow::Result<()> { let graph = Mutex::new({ let mut graph = IndexedTxGraph::new(index); - graph.apply_changeset(init_changeset.index_tx_graph); + graph.apply_changeset(init_changeset.indexed_tx_graph); graph }); @@ -277,7 +277,7 @@ fn main() -> anyhow::Result<()> { let chain = chain.apply_update(final_update.chain)?; - let index_tx_graph = { + let indexed_tx_graph = { let mut changeset = indexed_tx_graph::ChangeSet::::default(); let (_, indexer) = graph @@ -292,7 +292,7 @@ fn main() -> anyhow::Result<()> { }; ChangeSet { - index_tx_graph, + indexed_tx_graph, chain, } }; diff --git a/example-crates/example_esplora/Cargo.toml b/example-crates/example_esplora/Cargo.toml new file mode 100644 index 000000000..ccad862e9 --- /dev/null +++ b/example-crates/example_esplora/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "example_esplora" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bdk_chain = { path = "../../crates/chain", features = ["serde"] } +bdk_esplora = { path = "../../crates/esplora", features = ["blocking"] } +example_cli = { path = "../example_cli" } + diff --git a/example-crates/example_esplora/src/main.rs b/example-crates/example_esplora/src/main.rs new file mode 100644 index 000000000..f33125be8 --- /dev/null +++ b/example-crates/example_esplora/src/main.rs @@ -0,0 +1,324 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + io::{self, Write}, + sync::Mutex, +}; + +use bdk_chain::{ + bitcoin::{Address, Network, OutPoint, ScriptBuf, Txid}, + indexed_tx_graph::IndexedTxGraph, + keychain::WalletChangeSet, + local_chain::{CheckPoint, LocalChain}, + Append, ConfirmationTimeAnchor, +}; + +use bdk_esplora::{esplora_client, EsploraExt}; + +use example_cli::{ + anyhow::{self, Context}, + clap::{self, Parser, Subcommand}, + Keychain, +}; + +const DB_MAGIC: &[u8] = b"bdk_example_esplora"; +const DB_PATH: &str = ".bdk_esplora_example.db"; + +#[derive(Subcommand, Debug, Clone)] +enum EsploraCommands { + /// Scans the addresses in the wallet using the esplora API. + Scan { + /// When a gap this large has been found for a keychain, it will stop. + #[clap(long, default_value = "5")] + stop_gap: usize, + #[clap(flatten)] + scan_options: ScanOptions, + }, + /// Scan for particular addresses and unconfirmed transactions using the esplora API. + Sync { + /// Scan all the unused addresses. + #[clap(long)] + unused_spks: bool, + /// Scan every address that you have derived. + #[clap(long)] + all_spks: bool, + /// Scan unspent outpoints for spends or changes to confirmation status of residing tx. + #[clap(long)] + utxos: bool, + /// Scan unconfirmed transactions for updates. + #[clap(long)] + unconfirmed: bool, + #[clap(flatten)] + scan_options: ScanOptions, + }, +} + +#[derive(Parser, Debug, Clone, PartialEq)] +pub struct ScanOptions { + /// Max number of concurrent esplora server requests. + #[clap(long, default_value = "1")] + pub parallel_requests: usize, +} + +fn main() -> anyhow::Result<()> { + let (args, keymap, index, db, init_changeset) = example_cli::init::< + EsploraCommands, + WalletChangeSet, + >(DB_MAGIC, DB_PATH)?; + + // Contruct `IndexedTxGraph` and `LocalChain` with our initial changeset. They are wrapped in + // `Mutex` to display how they can be used in a multithreaded context. Technically the mutexes + // aren't strictly needed here. + let graph = Mutex::new({ + let mut graph = IndexedTxGraph::new(index); + graph.apply_changeset(init_changeset.indexed_tx_graph); + graph + }); + let chain = Mutex::new({ + let mut chain = LocalChain::default(); + chain.apply_changeset(&init_changeset.chain); + chain + }); + + let esplora_url = match args.network { + Network::Bitcoin => "https://blockstream.info/api", + Network::Testnet => "https://blockstream.info/testnet/api", + Network::Regtest => "http://localhost:3002", + Network::Signet => "https://mempool.space/signet/api", + _ => panic!("unsupported network"), + }; + + let client = esplora_client::Builder::new(esplora_url).build_blocking()?; + + let esplora_cmd = match &args.command { + // These are commands that are handled by this example (sync, scan). + example_cli::Commands::ChainSpecific(esplora_cmd) => esplora_cmd, + // These are general commands handled by example_cli. Execute the cmd and return. + general_cmd => { + let res = example_cli::handle_commands( + &graph, + &db, + &chain, + &keymap, + args.network, + |tx| { + client + .broadcast(tx) + .map(|_| ()) + .map_err(anyhow::Error::from) + }, + general_cmd.clone(), + ); + + db.lock().unwrap().commit()?; + return res; + } + }; + + // Prepare the `IndexedTxGraph` update based on whether we are scanning or syncing. + // Scanning: We are iterating through spks of all keychains and scanning for transactions for + // each spk. We start with the lowest derivation index spk and stop scanning after `stop_gap` + // number of consecutive spks have no transaction history. A Scan is done in situations of + // wallet restoration. It is a special case. Applications should use "sync" style updates + // after an initial scan. + // Syncing: We only check for specified spks, utxos and txids to update their confirmation + // status or fetch missing transactions. + let indexed_tx_graph_changeset = match &esplora_cmd { + EsploraCommands::Scan { + stop_gap, + scan_options, + } => { + let keychain_spks = graph + .lock() + .expect("mutex must not be poisoned") + .index + .spks_of_all_keychains() + .into_iter() + // This `map` is purely for logging. + .map(|(keychain, iter)| { + let mut first = true; + let spk_iter = iter.inspect(move |(i, _)| { + if first { + eprint!("\nscanning {}: ", keychain); + first = false; + } + eprint!("{} ", i); + // Flush early to ensure we print at every iteration. + let _ = io::stderr().flush(); + }); + (keychain, spk_iter) + }) + .collect::>(); + + // The client scans keychain spks for transaction histories, stopping after `stop_gap` + // is reached. It returns a `TxGraph` update (`graph_update`) and a structure that + // represents the last active spk derivation indices of keychains + // (`keychain_indices_update`). + let (graph_update, last_active_indices) = client + .update_tx_graph( + keychain_spks, + core::iter::empty(), + core::iter::empty(), + *stop_gap, + scan_options.parallel_requests, + ) + .context("scanning for transactions")?; + + let mut graph = graph.lock().expect("mutex must not be poisoned"); + // Because we did a stop gap based scan we are likely to have some updates to our + // deriviation indices. Usually before a scan you are on a fresh wallet with no + // addresses derived so we need to derive up to last active addresses the scan found + // before adding the transactions. + let (_, index_changeset) = graph.index.reveal_to_target_multi(&last_active_indices); + let mut indexed_tx_graph_changeset = graph.apply_update(graph_update); + indexed_tx_graph_changeset.append(index_changeset.into()); + indexed_tx_graph_changeset + } + EsploraCommands::Sync { + mut unused_spks, + all_spks, + mut utxos, + mut unconfirmed, + scan_options, + } => { + if !(*all_spks || unused_spks || utxos || unconfirmed) { + // If nothing is specifically selected, we select everything (except all spks). + unused_spks = true; + unconfirmed = true; + utxos = true; + } else if *all_spks { + // If all spks is selected, we don't need to also select unused spks (as unused spks + // is a subset of all spks). + unused_spks = false; + } + + // Spks, outpoints and txids we want updates on will be accumulated here. + let mut spks: Box> = Box::new(core::iter::empty()); + let mut outpoints: Box> = Box::new(core::iter::empty()); + let mut txids: Box> = Box::new(core::iter::empty()); + + // Get a short lock on the structures to get spks, utxos, and txs that we are interested + // in. + { + let graph = graph.lock().unwrap(); + let chain = chain.lock().unwrap(); + let chain_tip = chain.tip().map(|cp| cp.block_id()).unwrap_or_default(); + + if *all_spks { + let all_spks = graph + .index + .all_spks() + .iter() + .map(|(k, v)| (*k, v.clone())) + .collect::>(); + spks = Box::new(spks.chain(all_spks.into_iter().map(|(index, script)| { + eprintln!("scanning {:?}", index); + // Flush early to ensure we print at every iteration. + let _ = io::stderr().flush(); + script + }))); + } + if unused_spks { + let unused_spks = graph + .index + .unused_spks(..) + .map(|(k, v)| (*k, v.to_owned())) + .collect::>(); + spks = Box::new(spks.chain(unused_spks.into_iter().map(|(index, script)| { + eprintln!( + "Checking if address {} {:?} has been used", + Address::from_script(&script, args.network).unwrap(), + index + ); + // Flush early to ensure we print at every iteration. + let _ = io::stderr().flush(); + script + }))); + } + if utxos { + // We want to search for whether the UTXO is spent, and spent by which + // transaction. We provide the outpoint of the UTXO to + // `EsploraExt::update_tx_graph_without_keychain`. + let init_outpoints = graph.index.outpoints().iter().cloned(); + let utxos = graph + .graph() + .filter_chain_unspents(&*chain, chain_tip, init_outpoints) + .map(|(_, utxo)| utxo) + .collect::>(); + outpoints = Box::new( + utxos + .into_iter() + .inspect(|utxo| { + eprintln!( + "Checking if outpoint {} (value: {}) has been spent", + utxo.outpoint, utxo.txout.value + ); + // Flush early to ensure we print at every iteration. + let _ = io::stderr().flush(); + }) + .map(|utxo| utxo.outpoint), + ); + }; + if unconfirmed { + // We want to search for whether the unconfirmed transaction is now confirmed. + // We provide the unconfirmed txids to + // `EsploraExt::update_tx_graph_without_keychain`. + let unconfirmed_txids = graph + .graph() + .list_chain_txs(&*chain, chain_tip) + .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed()) + .map(|canonical_tx| canonical_tx.tx_node.txid) + .collect::>(); + txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| { + eprintln!("Checking if {} is confirmed yet", txid); + // Flush early to ensure we print at every iteration. + let _ = io::stderr().flush(); + })); + } + } + + let graph_update = client.update_tx_graph_without_keychain( + spks, + txids, + outpoints, + scan_options.parallel_requests, + )?; + + graph.lock().unwrap().apply_update(graph_update) + } + }; + + println!(); + + // Now that we're done updating the `IndexedTxGraph`, it's time to update the `LocalChain`! We + // want the `LocalChain` to have data about all the anchors in the `TxGraph` - for this reason, + // we want retrieve the blocks at the heights of the newly added anchors that are missing from + // our view of the chain. + let (missing_block_heights, tip) = { + let chain = &*chain.lock().unwrap(); + let missing_block_heights = indexed_tx_graph_changeset + .graph + .missing_heights_from(chain) + .collect::>(); + let tip = chain.tip(); + (missing_block_heights, tip) + }; + + println!("prev tip: {}", tip.as_ref().map_or(0, CheckPoint::height)); + println!("missing block heights: {:?}", missing_block_heights); + + // Here, we actually fetch the missing blocks and create a `local_chain::Update`. + let chain_update = client + .update_local_chain(tip, missing_block_heights) + .context("scanning for blocks")?; + + println!("new tip: {}", chain_update.tip.height()); + + // We persist the changes + let mut db = db.lock().unwrap(); + db.stage(WalletChangeSet { + chain: chain.lock().unwrap().apply_update(chain_update)?, + indexed_tx_graph: indexed_tx_graph_changeset, + }); + db.commit()?; + Ok(()) +}