Merge #1229: Use a universal lookahead value for KeychainTxOutIndex…

… and have a reasonable default

1def76f chore: make clippy happy and bump clippy msrv (志宇)
c9467dc chore: improve documentation of lookahead (LLFourn)
bc796f4 fix(example): bitcoind_rpc_polling now initializes local_chain properly (志宇)
4fd539b feat(chain)!: `KeychainTxOutIndex` uses a universal lookahead (Antoine Poinsot)

Pull request description:

  ### Description

  The `bdk::Wallet` is currently created without setting any lookahead value for the keychain. This implicitly makes it a lookahead of 0. As this is a high-level interface we should avoid footguns and aim for a reasonable default.

  To fix this, we have also decided to change `KeychainTxOutIndex` to have a default lookahead. Additionally, we have simplified the `KeychainTxOutIndex` API to have a single `lookahead` that is ONLY set at construction `KeychainTxOutIndex::new(lookahead: u32) -> Self`. This avoids the footguns of having methods which allows the caller to decrease the `lookahead` (which will panic).

  ### Notes to the reviewers

  ~A way to set this value externally is introduced in #1172. This PR only aims to use a saner default than 0. `1_000` is the value used by the Bitcoin Core wallet, and that seems reasonable to me.~

  Edit: we should NOT allow setting the `lookahead` value after-the-fact. Instead, the `lookahead` should be provided to the wallet's constructor.

  @evanlinjin: I don't think additional tests are necessary as no additional features are added, and the surface area of the API is decreased. The original tests already thoroughly test the `lookahead` concept.

  ### Checklists

  #### All Submissions:

  *(This section was updated as this PR changed from being a simple setting to introducing a new method.)*

  * [x] I've signed all my commits
  * [x] I followed the [contribution guidelines](https://github.com/bitcoindevkit/bdk/blob/master/CONTRIBUTING.md)
  * [x] I ran `cargo fmt` and `cargo clippy` before committing
  ~* [ ] I've added tests~
  * [x] I've added docs

ACKs for top commit:
  LLFourn:
    ACK 1def76f

Tree-SHA512: b4c3be8a4f2ac4877cf3f05852147e7dd1daeb02d3bc40895f02fd2a58e584f1dc0735b524153ff0875380ac93c0b4c31e516873d7a9b0027fdbbb5fe7970ff2

.github/workflows/cont_integration.yml

@@ -118,7 +118,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: "stable"
+            toolchain: stable
             components: clippy
             override: true
       - name: Rust Cache

clippy.toml

@@ -1 +1 @@
-msrv="1.57.0"
+msrv="1.63.0"

crates/bdk/src/descriptor/template.rs

@@ -575,7 +575,7 @@ mod test {
 
         if let ExtendedDescriptor::Pkh(pkh) = xdesc.0 {
             let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
-            let purpose = path.get(0).unwrap();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 0 });
@@ -589,7 +589,7 @@ mod test {
 
         if let ExtendedDescriptor::Pkh(pkh) = tdesc.0 {
             let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
-            let purpose = path.get(0).unwrap();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 1 });

crates/bdk/src/wallet/signer.rs

@@ -812,9 +812,10 @@ pub struct SignOptions {
 }
 
 /// Customize which taproot script-path leaves the signer should sign.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub enum TapLeavesOptions {
     /// The signer will sign all the leaves it has a key for.
+    #[default]
     All,
     /// The signer won't sign leaves other than the ones specified. Note that it could still ignore
     /// some of the specified leaves, if it doesn't have the right key to sign them.
@@ -825,12 +826,6 @@ pub enum TapLeavesOptions {
     None,
 }
 
-impl Default for TapLeavesOptions {
-    fn default() -> Self {
-        TapLeavesOptions::All
-    }
-}
-
 #[allow(clippy::derivable_impls)]
 impl Default for SignOptions {
     fn default() -> Self {

crates/bdk/src/wallet/tx_builder.rs

@@ -811,22 +811,17 @@ impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
 }
 
 /// Ordering of the transaction's inputs and outputs
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum TxOrdering {
     /// Randomized (default)
+    #[default]
     Shuffle,
     /// Unchanged
     Untouched,
     /// BIP69 / Lexicographic
     Bip69Lexicographic,
 }
 
-impl Default for TxOrdering {
-    fn default() -> Self {
-        TxOrdering::Shuffle
-    }
-}
-
 impl TxOrdering {
     /// Sort transaction inputs and outputs by [`TxOrdering`] variant
     pub fn sort_tx(&self, tx: &mut Transaction) {
@@ -880,22 +875,17 @@ impl RbfValue {
 }
 
 /// Policy regarding the use of change outputs when creating a transaction
-#[derive(Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
 pub enum ChangeSpendPolicy {
     /// Use both change and non-change outputs (default)
+    #[default]
     ChangeAllowed,
     /// Only use change outputs (see [`TxBuilder::only_spend_change`])
     OnlyChange,
     /// Only use non-change outputs (see [`TxBuilder::do_not_spend_change`])
     ChangeForbidden,
 }
 
-impl Default for ChangeSpendPolicy {
-    fn default() -> Self {
-        ChangeSpendPolicy::ChangeAllowed
-    }
-}
-
 impl ChangeSpendPolicy {
     pub(crate) fn is_satisfied_by(&self, utxo: &LocalOutput) -> bool {
         match self {

crates/chain/src/keychain/txout_index.rs

@@ -5,12 +5,13 @@ use crate::{
     spk_iter::BIP32_MAX_INDEX,
     SpkIterator, SpkTxOutIndex,
 };
-use alloc::vec::Vec;
 use bitcoin::{OutPoint, Script, TxOut};
 use core::{fmt::Debug, ops::Deref};
 
 use crate::Append;
 
+const DEFAULT_LOOKAHEAD: u32 = 1_000;
+
 /// A convenient wrapper around [`SpkTxOutIndex`] that relates script pubkeys to miniscript public
 /// [`Descriptor`]s.
 ///
@@ -46,7 +47,7 @@ use crate::Append;
 /// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
 /// # let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
 /// # let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
-/// # let descriptor_for_user_42 = external_descriptor.clone();
+/// # let (descriptor_for_user_42, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/2/*)").unwrap();
 /// txout_index.add_keychain(MyKeychain::External, external_descriptor);
 /// txout_index.add_keychain(MyKeychain::Internal, internal_descriptor);
 /// txout_index.add_keychain(MyKeychain::MyAppUser { user_id: 42 }, descriptor_for_user_42);
@@ -65,17 +66,12 @@ pub struct KeychainTxOutIndex<K> {
     // last revealed indexes
     last_revealed: BTreeMap<K, u32>,
     // lookahead settings for each keychain
-    lookahead: BTreeMap<K, u32>,
+    lookahead: u32,
 }
 
 impl<K> Default for KeychainTxOutIndex<K> {
     fn default() -> Self {
-        Self {
-            inner: SpkTxOutIndex::default(),
-            keychains: BTreeMap::default(),
-            last_revealed: BTreeMap::default(),
-            lookahead: BTreeMap::default(),
-        }
+        Self::new(DEFAULT_LOOKAHEAD)
     }
 }
 
@@ -118,6 +114,25 @@ impl<K: Clone + Ord + Debug> Indexer for KeychainTxOutIndex<K> {
     }
 }
 
+impl<K> KeychainTxOutIndex<K> {
+    /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`.
+    ///
+    /// The `lookahead` is the number of script pubkeys to derive and cache from the internal
+    /// descriptors over and above the last revealed script index. Without a lookahead the index
+    /// will miss outputs you own when processing transactions whose output script pubkeys lie
+    /// beyond the last revealed index. In certain situations, such as when performing an initial
+    /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index
+    /// of the last revealed script pubkey actually is.
+    pub fn new(lookahead: u32) -> Self {
+        Self {
+            inner: SpkTxOutIndex::default(),
+            keychains: BTreeMap::new(),
+            last_revealed: BTreeMap::new(),
+            lookahead,
+        }
+    }
+}
+
 impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     /// Return a reference to the internal [`SpkTxOutIndex`].
     pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
@@ -145,54 +160,22 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn add_keychain(&mut self, keychain: K, descriptor: Descriptor<DescriptorPublicKey>) {
         let old_descriptor = &*self
             .keychains
-            .entry(keychain)
+            .entry(keychain.clone())
             .or_insert_with(|| descriptor.clone());
         assert_eq!(
             &descriptor, old_descriptor,
             "keychain already contains a different descriptor"
         );
+        self.replenish_lookahead(&keychain, self.lookahead);
     }
 
-    /// Return the lookahead setting for each keychain.
+    /// Get the lookahead setting.
     ///
-    /// Refer to [`set_lookahead`] for a deeper explanation of the `lookahead`.
+    /// Refer to [`new`] for more information on the `lookahead`.
     ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn lookaheads(&self) -> &BTreeMap<K, u32> {
-        &self.lookahead
-    }
-
-    /// Convenience method to call [`set_lookahead`] for all keychains.
-    ///
-    /// [`set_lookahead`]: Self::set_lookahead
-    pub fn set_lookahead_for_all(&mut self, lookahead: u32) {
-        for keychain in &self.keychains.keys().cloned().collect::<Vec<_>>() {
-            self.set_lookahead(keychain, lookahead);
-        }
-    }
-
-    /// Set the lookahead count for `keychain`.
-    ///
-    /// The lookahead is the number of scripts to cache ahead of the last revealed script index. This
-    /// is useful to find outputs you own when processing block data that lie beyond the last revealed
-    /// index. In certain situations, such as when performing an initial scan of the blockchain during
-    /// wallet import, it may be uncertain or unknown what the last revealed index is.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the `keychain` does not exist.
-    pub fn set_lookahead(&mut self, keychain: &K, lookahead: u32) {
-        self.lookahead.insert(keychain.clone(), lookahead);
-        self.replenish_lookahead(keychain);
-    }
-
-    /// Convenience method to call [`lookahead_to_target`] for multiple keychains.
-    ///
-    /// [`lookahead_to_target`]: Self::lookahead_to_target
-    pub fn lookahead_to_target_multi(&mut self, target_indexes: BTreeMap<K, u32>) {
-        for (keychain, target_index) in target_indexes {
-            self.lookahead_to_target(&keychain, target_index)
-        }
+    /// [`new`]: Self::new
+    pub fn lookahead(&self) -> u32 {
+        self.lookahead
     }
 
     /// Store lookahead scripts until `target_index`.
@@ -201,22 +184,14 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
     pub fn lookahead_to_target(&mut self, keychain: &K, target_index: u32) {
         let next_index = self.next_store_index(keychain);
         if let Some(temp_lookahead) = target_index.checked_sub(next_index).filter(|&v| v > 0) {
-            let old_lookahead = self.lookahead.insert(keychain.clone(), temp_lookahead);
-            self.replenish_lookahead(keychain);
-
-            // revert
-            match old_lookahead {
-                Some(lookahead) => self.lookahead.insert(keychain.clone(), lookahead),
-                None => self.lookahead.remove(keychain),
-            };
+            self.replenish_lookahead(keychain, temp_lookahead);
         }
     }
 
-    fn replenish_lookahead(&mut self, keychain: &K) {
+    fn replenish_lookahead(&mut self, keychain: &K, lookahead: u32) {
         let descriptor = self.keychains.get(keychain).expect("keychain must exist");
         let next_store_index = self.next_store_index(keychain);
         let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
 
         for (new_index, new_spk) in
             SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead)
@@ -388,25 +363,21 @@ impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
 
         let target_index = if has_wildcard { target_index } else { 0 };
         let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-        let lookahead = self.lookahead.get(keychain).map_or(0, |v| *v);
 
-        debug_assert_eq!(
-            next_reveal_index + lookahead,
-            self.next_store_index(keychain)
-        );
+        debug_assert!(next_reveal_index + self.lookahead >= self.next_store_index(keychain));
 
         // if we need to reveal new indices, the latest revealed index goes here
         let mut reveal_to_index = None;
 
         // if the target is not yet revealed, but is already stored (due to lookahead), we need to
         // set the `reveal_to_index` as target here (as the `for` loop below only updates
         // `reveal_to_index` for indexes that are NOT stored)
-        if next_reveal_index <= target_index && target_index < next_reveal_index + lookahead {
+        if next_reveal_index <= target_index && target_index < next_reveal_index + self.lookahead {
             reveal_to_index = Some(target_index);
         }
 
         // we range over indexes that are not stored
-        let range = next_reveal_index + lookahead..=target_index + lookahead;
+        let range = next_reveal_index + self.lookahead..=target_index + self.lookahead;
         for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
             let _inserted = self
                 .inner

crates/chain/src/spk_iter.rs

@@ -148,7 +148,7 @@ mod test {
         Descriptor<DescriptorPublicKey>,
         Descriptor<DescriptorPublicKey>,
     ) {
-        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
         let secp = Secp256k1::signing_only();
         let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();

crates/chain/src/spk_txout_index.rs

@@ -168,9 +168,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// Returns `None` if the `TxOut` hasn't been scanned or if nothing matching was found there.
     pub fn txout(&self, outpoint: OutPoint) -> Option<(&I, &TxOut)> {
-        self.txouts
-            .get(&outpoint)
-            .map(|(spk_i, txout)| (spk_i, txout))
+        self.txouts.get(&outpoint).map(|v| (&v.0, &v.1))
     }
 
     /// Returns the script that has been inserted at the `index`.

crates/chain/src/tx_graph.rs

@@ -581,10 +581,7 @@ impl<A: Clone + Ord> TxGraph<A> {
         }
 
         for (outpoint, txout) in changeset.txouts {
-            let tx_entry = self
-                .txs
-                .entry(outpoint.txid)
-                .or_insert_with(Default::default);
+            let tx_entry = self.txs.entry(outpoint.txid).or_default();
 
             match tx_entry {
                 (TxNodeInternal::Whole(_), _, _) => { /* do nothing since we already have full tx */
@@ -597,13 +594,13 @@ impl<A: Clone + Ord> TxGraph<A> {
 
         for (anchor, txid) in changeset.anchors {
             if self.anchors.insert((anchor.clone(), txid)) {
-                let (_, anchors, _) = self.txs.entry(txid).or_insert_with(Default::default);
+                let (_, anchors, _) = self.txs.entry(txid).or_default();
                 anchors.insert(anchor);
             }
         }
 
         for (txid, new_last_seen) in changeset.last_seen {
-            let (_, _, last_seen) = self.txs.entry(txid).or_insert_with(Default::default);
+            let (_, _, last_seen) = self.txs.entry(txid).or_default();
             if new_last_seen > *last_seen {
                 *last_seen = new_last_seen;
             }

crates/chain/tests/common/mod.rs

@@ -1,4 +1,5 @@
 mod tx_template;
+#[allow(unused_imports)]
 pub use tx_template::*;
 
 #[allow(unused_macros)]

crates/chain/tests/test_indexed_tx_graph.rs

@@ -27,9 +27,10 @@ fn insert_relevant_txs() {
     let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
     let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::new(
+        KeychainTxOutIndex::new(10),
+    );
     graph.index.add_keychain((), descriptor);
-    graph.index.set_lookahead(&(), 10);
 
     let tx_a = Transaction {
         output: vec![
@@ -118,12 +119,12 @@ fn test_list_owned_txouts() {
     let (desc_1, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
     let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)").unwrap();
 
-    let mut graph =
-        IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::new(
+        KeychainTxOutIndex::new(10),
+    );
 
     graph.index.add_keychain("keychain_1".into(), desc_1);
     graph.index.add_keychain("keychain_2".into(), desc_2);
-    graph.index.set_lookahead_for_all(10);
 
     // Get trusted and untrusted addresses