feat!: Rework sqlite, changesets, persistence and wallet-construction

Rework sqlite: Instead of only supported one schema (defined in
`bdk_sqlite`), we have a schema per changeset type for more flexiblity.

* rm `bdk_sqlite` crate (as we don't need `bdk_sqlite::Store` anymore).
* add `sqlite` feature on `bdk_chain` which adds methods on each
  changeset type for initializing tables, loading the changeset and
  writing.

Rework changesets: Some callers may want to use `KeychainTxOutIndex`
where `K` may change per descriptor on every run. So we only want to
persist the last revealed indices by `DescriptorId` (which uniquely-ish
identifies the descriptor).

* rm `keychain_added` field from `keychain_txout`'s changeset.
* Add `keychain_added` to `CombinedChangeSet` (which is renamed to
  `WalletChangeSet`).

Rework persistence: add back some safety and convenience when persisting
our types. Working with changeset directly (as we were doing before) can
be cumbersome.

* Intoduce `struct Persisted<T>` which wraps a type `T` which stores
  staged changes to it. This adds safety when creating and or loading
  `T` from db.
* `struct Persisted<T>` methods, `create`, `load` and `persist`, are
  avaliable if `trait PersistWith<Db>` is implemented for `T`. `Db`
  represents the database connection and `PersistWith` should be
  implemented per database-type.
* For async, we have `trait PersistedAsyncWith<Db>`.
* `Wallet` has impls of `PersistedWith<rusqlite::Connection>`,
  `PersistedWith<rusqlite::Transaction>` and
  `PersistedWith<bdk_file_store::Store>` by default.

Rework wallet-construction: Before, we had multiple methods for loading
and creating with different input-counts so it would be unwieldly to add
more parameters in the future. This also makes it difficult to impl
`PersistWith` (which has a single method for `load` that takes in
`PersistWith::LoadParams` and a single method for `create` that takes in
`PersistWith::CreateParams`).

* Introduce a builder pattern when constructing a `Wallet`. For loading
  from persistence or `ChangeSet`, we have `LoadParams`. For creating a
  new wallet, we have `CreateParams`.

Cargo.toml

@@ -4,7 +4,6 @@ members = [
     "crates/wallet",
     "crates/chain",
     "crates/file_store",
-    "crates/sqlite",
     "crates/electrum",
     "crates/esplora",
     "crates/bitcoind_rpc",

crates/bitcoind_rpc/tests/test_emitter.rs

@@ -4,7 +4,8 @@ use bdk_bitcoind_rpc::Emitter;
 use bdk_chain::{
     bitcoin::{Address, Amount, Txid},
     local_chain::{CheckPoint, LocalChain},
-    Balance, BlockId, IndexedTxGraph, Merge, SpkTxOutIndex,
+    spk_txout::SpkTxOutIndex,
+    Balance, BlockId, IndexedTxGraph, Merge,
 };
 use bdk_testenv::{anyhow, TestEnv};
 use bitcoin::{hashes::Hash, Block, OutPoint, ScriptBuf, WScriptHash};
@@ -47,7 +48,7 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
 
         assert_eq!(
             local_chain.apply_update(emission.checkpoint,)?,
-            BTreeMap::from([(height, Some(hash))]),
+            [(height, Some(hash))].into(),
             "chain update changeset is unexpected",
         );
     }
@@ -93,11 +94,13 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
         assert_eq!(
             local_chain.apply_update(emission.checkpoint,)?,
             if exp_height == exp_hashes.len() - reorged_blocks.len() {
-                core::iter::once((height, Some(hash)))
-                    .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
-                    .collect::<bdk_chain::local_chain::ChangeSet>()
+                bdk_chain::local_chain::ChangeSet {
+                    blocks: core::iter::once((height, Some(hash)))
+                        .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
+                        .collect(),
+                }
             } else {
-                BTreeMap::from([(height, Some(hash))])
+                [(height, Some(hash))].into()
             },
             "chain update changeset is unexpected",
         );
@@ -193,15 +196,15 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
         let indexed_additions = indexed_tx_graph.batch_insert_unconfirmed(mempool_txs);
         assert_eq!(
             indexed_additions
-                .graph
+                .tx_graph
                 .txs
                 .iter()
                 .map(|tx| tx.compute_txid())
                 .collect::<BTreeSet<Txid>>(),
             exp_txids,
             "changeset should have the 3 mempool transactions",
         );
-        assert!(indexed_additions.graph.anchors.is_empty());
+        assert!(indexed_additions.tx_graph.anchors.is_empty());
     }
 
     // mine a block that confirms the 3 txs
@@ -224,9 +227,9 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
         let height = emission.block_height();
         let _ = chain.apply_update(emission.checkpoint)?;
         let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
-        assert!(indexed_additions.graph.txs.is_empty());
-        assert!(indexed_additions.graph.txouts.is_empty());
-        assert_eq!(indexed_additions.graph.anchors, exp_anchors);
+        assert!(indexed_additions.tx_graph.txs.is_empty());
+        assert!(indexed_additions.tx_graph.txouts.is_empty());
+        assert_eq!(indexed_additions.tx_graph.anchors, exp_anchors);
     }
 
     Ok(())

crates/chain/Cargo.toml

@@ -20,6 +20,10 @@ serde_crate = { package = "serde", version = "1", optional = true, features = ["
 hashbrown = { version = "0.9.1", optional = true, features = ["serde"] }
 miniscript = { version = "12.0.0", optional = true, default-features = false }
 
+# Feature dependencies
+rusqlite = { version = "0.31.0", features = ["bundled"], optional = true }
+serde_json = {version = "1", optional = true }
+
 [dev-dependencies]
 rand = "0.8"
 proptest = "1.2.0"
@@ -28,3 +32,4 @@ proptest = "1.2.0"
 default = ["std", "miniscript"]
 std = ["bitcoin/std", "miniscript?/std"]
 serde = ["serde_crate", "bitcoin/serde", "miniscript?/serde"]
+sqlite = ["std", "rusqlite", "serde", "serde_json"]

crates/chain/src/changeset.rs

@@ -1,92 +1,206 @@
+use crate::{ConfirmationBlockTime, Merge};
+
+type IndexedTxGraphChangeSet =
+    crate::indexed_tx_graph::ChangeSet<ConfirmationBlockTime, crate::keychain_txout::ChangeSet>;
+
 /// A changeset containing [`crate`] structures typically persisted together.
-#[cfg(feature = "miniscript")]
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq)]
 #[cfg_attr(
     feature = "serde",
     derive(crate::serde::Deserialize, crate::serde::Serialize),
-    serde(
-        crate = "crate::serde",
-        bound(
-            deserialize = "A: Ord + crate::serde::Deserialize<'de>, K: Ord + crate::serde::Deserialize<'de>",
-            serialize = "A: Ord + crate::serde::Serialize, K: Ord + crate::serde::Serialize",
-        ),
-    )
+    serde(crate = "crate::serde")
 )]
-pub struct CombinedChangeSet<K, A> {
-    /// Changes to the [`LocalChain`](crate::local_chain::LocalChain).
-    pub chain: crate::local_chain::ChangeSet,
-    /// Changes to [`IndexedTxGraph`](crate::indexed_tx_graph::IndexedTxGraph).
-    pub indexed_tx_graph:
-        crate::indexed_tx_graph::ChangeSet<A, crate::indexer::keychain_txout::ChangeSet<K>>,
+pub struct WalletChangeSet {
+    /// Descriptor for recipient addresses.
+    pub descriptor: Option<miniscript::Descriptor<miniscript::DescriptorPublicKey>>,
+    /// Descriptor for change addresses.
+    pub change_descriptor: Option<miniscript::Descriptor<miniscript::DescriptorPublicKey>>,
     /// Stores the network type of the transaction data.
     pub network: Option<bitcoin::Network>,
+    /// Changes to the [`LocalChain`](crate::local_chain::LocalChain).
+    pub local_chain: crate::local_chain::ChangeSet,
+    /// Changes to [`TxGraph`](crate::tx_graph::TxGraph).
+    pub tx_graph: crate::tx_graph::ChangeSet<crate::ConfirmationBlockTime>,
+    /// Changes to [`KeychainTxOutIndex`](crate::keychain_txout::KeychainTxOutIndex).
+    pub indexer: crate::keychain_txout::ChangeSet,
 }
 
-#[cfg(feature = "miniscript")]
-impl<K, A> core::default::Default for CombinedChangeSet<K, A> {
-    fn default() -> Self {
-        Self {
-            chain: core::default::Default::default(),
-            indexed_tx_graph: core::default::Default::default(),
-            network: None,
-        }
-    }
-}
-
-#[cfg(feature = "miniscript")]
-impl<K: Ord, A: crate::Anchor> crate::Merge for CombinedChangeSet<K, A> {
+impl Merge for WalletChangeSet {
+    /// Merge another [`WalletChangeSet`] into itself.
+    ///
+    /// The `keychains_added` field respects the invariants of... TODO: FINISH THIS!
     fn merge(&mut self, other: Self) {
-        crate::Merge::merge(&mut self.chain, other.chain);
-        crate::Merge::merge(&mut self.indexed_tx_graph, other.indexed_tx_graph);
+        if other.descriptor.is_some() {
+            debug_assert!(
+                self.descriptor.is_none() || self.descriptor == other.descriptor,
+                "descriptor must never change"
+            );
+            self.descriptor = other.descriptor;
+        }
+        if other.change_descriptor.is_some() {
+            debug_assert!(
+                self.change_descriptor.is_none()
+                    || self.change_descriptor == other.change_descriptor,
+                "change descriptor must never change"
+            );
+        }
         if other.network.is_some() {
             debug_assert!(
                 self.network.is_none() || self.network == other.network,
-                "network type must either be just introduced or remain the same"
+                "network must never change"
             );
             self.network = other.network;
         }
+
+        crate::Merge::merge(&mut self.local_chain, other.local_chain);
+        crate::Merge::merge(&mut self.tx_graph, other.tx_graph);
+        crate::Merge::merge(&mut self.indexer, other.indexer);
     }
 
     fn is_empty(&self) -> bool {
-        self.chain.is_empty() && self.indexed_tx_graph.is_empty() && self.network.is_none()
+        self.descriptor.is_none()
+            && self.change_descriptor.is_none()
+            && self.network.is_none()
+            && self.local_chain.is_empty()
+            && self.tx_graph.is_empty()
+            && self.indexer.is_empty()
+    }
+}
+
+#[cfg(feature = "sqlite")]
+impl WalletChangeSet {
+    /// Schema name for wallet.
+    pub const WALLET_SCHEMA_NAME: &'static str = "bdk_wallet";
+    /// Name of table to store wallet descriptors and network.
+    pub const WALLET_TABLE_NAME: &'static str = "bdk_wallet";
+
+    /// Initialize sqlite tables for wallet schema & table.
+    fn init_wallet_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        let schema_v0: &[&str] = &[&format!(
+            "CREATE TABLE {} ( \
+                id INTEGER PRIMARY KEY NOT NULL CHECK (id = 0), \
+                descriptor TEXT, \
+                change_descriptor TEXT, \
+                network TEXT \
+                ) STRICT;",
+            Self::WALLET_TABLE_NAME,
+        )];
+        crate::sqlite::migrate_schema(db_tx, Self::WALLET_SCHEMA_NAME, &[schema_v0])
+    }
+
+    /// Recover a [`WalletChangeSet`] from sqlite database.
+    pub fn from_sqlite(db_tx: &rusqlite::Transaction) -> rusqlite::Result<Self> {
+        Self::init_wallet_sqlite_tables(db_tx)?;
+        use crate::sqlite::Sql;
+        use miniscript::{Descriptor, DescriptorPublicKey};
+        use rusqlite::OptionalExtension;
+
+        let mut changeset = Self::default();
+
+        let mut wallet_statement = db_tx.prepare(&format!(
+            "SELECT descriptor, change_descriptor, network FROM {}",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        let row = wallet_statement
+            .query_row([], |row| {
+                Ok((
+                    row.get::<_, Sql<Descriptor<DescriptorPublicKey>>>("descriptor")?,
+                    row.get::<_, Sql<Descriptor<DescriptorPublicKey>>>("change_descriptor")?,
+                    row.get::<_, Sql<bitcoin::Network>>("network")?,
+                ))
+            })
+            .optional()?;
+        if let Some((Sql(desc), Sql(change_desc), Sql(network))) = row {
+            changeset.descriptor = Some(desc);
+            changeset.change_descriptor = Some(change_desc);
+            changeset.network = Some(network);
+        }
+
+        changeset.local_chain = crate::local_chain::ChangeSet::from_sqlite(db_tx)?;
+        changeset.tx_graph = crate::tx_graph::ChangeSet::<_>::from_sqlite(db_tx)?;
+        changeset.indexer = crate::indexer::keychain_txout::ChangeSet::from_sqlite(db_tx)?;
+
+        Ok(changeset)
+    }
+
+    /// Persist [`WalletChangeSet`] to sqlite database.
+    pub fn persist_to_sqlite(&self, db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        Self::init_wallet_sqlite_tables(db_tx)?;
+        use crate::sqlite::Sql;
+        use rusqlite::named_params;
+
+        let mut descriptor_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, descriptor) VALUES(:id, :descriptor) ON CONFLICT(id) DO UPDATE SET descriptor=:descriptor",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(descriptor) = &self.descriptor {
+            descriptor_statement.execute(named_params! {
+                ":id": 0,
+                ":descriptor": Sql(descriptor.clone()),
+            })?;
+        }
+
+        let mut change_descriptor_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, change_descriptor) VALUES(:id, :change_descriptor) ON CONFLICT(id) DO UPDATE SET change_descriptor=:change_descriptor",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(change_descriptor) = &self.change_descriptor {
+            change_descriptor_statement.execute(named_params! {
+                ":id": 0,
+                ":change_descriptor": Sql(change_descriptor.clone()),
+            })?;
+        }
+
+        let mut network_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, network) VALUES(:id, :network) ON CONFLICT(id) DO UPDATE SET network=:network",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(network) = self.network {
+            network_statement.execute(named_params! {
+                ":id": 0,
+                ":network": Sql(network),
+            })?;
+        }
+
+        self.local_chain.persist_to_sqlite(db_tx)?;
+        self.tx_graph.persist_to_sqlite(db_tx)?;
+        self.indexer.persist_to_sqlite(db_tx)?;
+        Ok(())
     }
 }
 
-#[cfg(feature = "miniscript")]
-impl<K, A> From<crate::local_chain::ChangeSet> for CombinedChangeSet<K, A> {
+impl From<crate::local_chain::ChangeSet> for WalletChangeSet {
     fn from(chain: crate::local_chain::ChangeSet) -> Self {
         Self {
-            chain,
+            local_chain: chain,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<IndexedTxGraphChangeSet> for WalletChangeSet {
+    fn from(indexed_tx_graph: IndexedTxGraphChangeSet) -> Self {
+        Self {
+            tx_graph: indexed_tx_graph.tx_graph,
+            indexer: indexed_tx_graph.indexer,
             ..Default::default()
         }
     }
 }
 
-#[cfg(feature = "miniscript")]
-impl<K, A> From<crate::indexed_tx_graph::ChangeSet<A, crate::indexer::keychain_txout::ChangeSet<K>>>
-    for CombinedChangeSet<K, A>
-{
-    fn from(
-        indexed_tx_graph: crate::indexed_tx_graph::ChangeSet<
-            A,
-            crate::indexer::keychain_txout::ChangeSet<K>,
-        >,
-    ) -> Self {
+impl From<crate::tx_graph::ChangeSet<ConfirmationBlockTime>> for WalletChangeSet {
+    fn from(tx_graph: crate::tx_graph::ChangeSet<ConfirmationBlockTime>) -> Self {
         Self {
-            indexed_tx_graph,
+            tx_graph,
             ..Default::default()
         }
     }
 }
 
-#[cfg(feature = "miniscript")]
-impl<K, A> From<crate::indexer::keychain_txout::ChangeSet<K>> for CombinedChangeSet<K, A> {
-    fn from(indexer: crate::indexer::keychain_txout::ChangeSet<K>) -> Self {
+impl From<crate::keychain_txout::ChangeSet> for WalletChangeSet {
+    fn from(indexer: crate::keychain_txout::ChangeSet) -> Self {
         Self {
-            indexed_tx_graph: crate::indexed_tx_graph::ChangeSet {
-                indexer,
-                ..Default::default()
-            },
+            indexer,
             ..Default::default()
         }
     }