Merge bitcoindevkit/bdk#1522: Bump bdk version to 1.0.0-beta.1

53bea0d902 Bump bdk version to 1.0.0-beta.1 (Steve Myers)

Pull request description:

  ### Description

  Bump bdk version to 1.0.0-beta.1

  bdk_chain to 0.17.0
  bdk_bitcoind_rpc to 0.13.0
  bdk_electrum to 0.16.0
  bdk_esplora to 0.16.0
  bdk_file_store to 0.14.0
  bdk_testenv to 0.7.0
  bdk_hwi to 0.4.0

  ### Notes to the reviewers

  This release completes the 1.0.0-alpha milestone and starts our new 1.0.0-beta milestone series of releases.

  The beta releases will maintain a stable `bdk_wallet` API and be focused on improving testing, docs, CI, and fixing any newly discovered bugs.

ACKs for top commit:
  LLFourn:
    ACK 53bea0d902
  LagginTimes:
    ACK 53bea0d902
  storopoli:
    ACK 53bea0d902

Tree-SHA512: cc2adbdeb7c2786659198da64e96fcb374833666e8d98c3f255e2de5f7283573efe3e3c53fd0c7b0e475fe3ac30d3c88169075e079d4d0b67496fc9a962b39db

.github/workflows/cont_integration.yml

@@ -32,8 +32,10 @@ jobs:
         run: |
           cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
           cargo update -p time --precise "0.3.20"
-          cargo update -p jobserver --precise "0.1.26"
           cargo update -p home --precise "0.5.5"
+          cargo update -p proptest --precise "1.2.0"
+          cargo update -p url --precise "2.5.0"
+          cargo update -p cc --precise "1.0.105"
       - name: Build
         run: cargo build ${{ matrix.features }}
       - name: Test
@@ -57,15 +59,15 @@ jobs:
       - name: Check bdk_chain
         working-directory: ./crates/chain
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,hashbrown
-      - name: Check bdk
-        working-directory: ./crates/bdk
+        run: cargo check --no-default-features --features miniscript/no-std,hashbrown
+      - name: Check bdk wallet
+        working-directory: ./crates/wallet
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown
+        run: cargo check --no-default-features --features miniscript/no-std,bdk_chain/hashbrown
       - name: Check esplora
         working-directory: ./crates/esplora
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown
+        run: cargo check --no-default-features --features miniscript/no-std,bdk_chain/hashbrown
 
   check-wasm:
     name: Check WASM
@@ -89,12 +91,12 @@ jobs:
             target: "wasm32-unknown-unknown"
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.2.1
-      - name: Check bdk
-        working-directory: ./crates/bdk
-        run: cargo check --target wasm32-unknown-unknown --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
+      - name: Check bdk wallet
+        working-directory: ./crates/wallet
+        run: cargo check --target wasm32-unknown-unknown --no-default-features --features miniscript/no-std,bdk_chain/hashbrown
       - name: Check esplora
         working-directory: ./crates/esplora
-        run: cargo check --target wasm32-unknown-unknown --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,async
+        run: cargo check --target wasm32-unknown-unknown --no-default-features --features miniscript/no-std,bdk_chain/hashbrown,async
 
   fmt:
     name: Rust fmt
@@ -118,7 +120,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            toolchain: stable
+            toolchain: 1.78.0
             components: clippy
             override: true
       - name: Rust Cache

.github/workflows/nightly_docs.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
       - name: Set default toolchain
-        run: rustup default nightly-2022-12-14
+        run: rustup default nightly-2024-05-12
       - name: Set profile
         run: rustup set profile minimal
       - name: Update toolchain

.gitignore

@@ -7,3 +7,4 @@ Cargo.lock
 
 # Example persisted files.
 *.db
+*.sqlite*

Cargo.toml

@@ -1,13 +1,14 @@
 [workspace]
 resolver = "2"
 members = [
-    "crates/bdk",
+    "crates/wallet",
     "crates/chain",
     "crates/file_store",
     "crates/electrum",
     "crates/esplora",
     "crates/bitcoind_rpc",
     "crates/hwi",
+    "crates/testenv",
     "example-crates/example_cli",
     "example-crates/example_electrum",
     "example-crates/example_esplora",

README.md

@@ -10,11 +10,11 @@
   </p>
 
   <p>
-    <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
+    <a href="https://crates.io/crates/bdk_wallet"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk_wallet.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
-    <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
+    <a href="https://docs.rs/bdk_wallet"><img alt="Wallet API Docs" src="https://img.shields.io/badge/docs.rs-bdk_wallet-green"/></a>
     <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
@@ -22,7 +22,7 @@
   <h4>
     <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://docs.rs/bdk">Documentation</a>
+    <a href="https://docs.rs/bdk_wallet">Documentation</a>
   </h4>
 </div>
 
@@ -39,17 +39,18 @@ It is built upon the excellent [`rust-bitcoin`] and [`rust-miniscript`] crates.
 
 The project is split up into several crates in the `/crates` directory:
 
-- [`bdk`](./crates/bdk): Contains the central high level `Wallet` type that is built from the low-level mechanisms provided by the other components
+- [`wallet`](./crates/wallet): Contains the central high level `Wallet` type that is built from the low-level mechanisms provided by the other components
 - [`chain`](./crates/chain): Tools for storing and indexing chain data
+- [`persist`](./crates/persist): Types that define data persistence of a BDK wallet
 - [`file_store`](./crates/file_store): A (experimental) persistence backend for storing chain data in a single file.
 - [`esplora`](./crates/esplora): Extends the [`esplora-client`] crate with methods to fetch chain data from an esplora HTTP server in the form that [`bdk_chain`] and `Wallet` can consume.
 - [`electrum`](./crates/electrum): Extends the [`electrum-client`] crate with methods to fetch chain data from an electrum server in the form that [`bdk_chain`] and `Wallet` can consume.
 
 Fully working examples of how to use these components are in `/example-crates`:
-- [`example_cli`](./example-crates/example_cli): Library used by the `example_*` crates. Provides utilities for syncing, showing the balance, generating addresses and creating transactions without using the bdk `Wallet`.
-- [`example_electrum`](./example-crates/example_electrum): A command line Bitcoin wallet application built on top of `example_cli` and the `electrum` crate. It shows the power of the bdk tools (`chain` + `file_store` + `electrum`), without depending on the main `bdk` library.
-- [`example_esplora`](./example-crates/example_esplora): A command line Bitcoin wallet application built on top of `example_cli` and the `esplora` crate. It shows the power of the bdk tools (`chain` + `file_store` + `esplora`), without depending on the main `bdk` library.
-- [`example_bitcoind_rpc_polling`](./example-crates/example_bitcoind_rpc_polling): A command line Bitcoin wallet application built on top of `example_cli` and the `bitcoind_rpc` crate. It shows the power of the bdk tools (`chain` + `file_store` + `bitcoind_rpc`), without depending on the main `bdk` library.
+- [`example_cli`](./example-crates/example_cli): Library used by the `example_*` crates. Provides utilities for syncing, showing the balance, generating addresses and creating transactions without using the bdk_wallet `Wallet`.
+- [`example_electrum`](./example-crates/example_electrum): A command line Bitcoin wallet application built on top of `example_cli` and the `electrum` crate. It shows the power of the bdk tools (`chain` + `file_store` + `electrum`), without depending on the main `bdk_wallet` library.
+- [`example_esplora`](./example-crates/example_esplora): A command line Bitcoin wallet application built on top of `example_cli` and the `esplora` crate. It shows the power of the bdk tools (`chain` + `file_store` + `esplora`), without depending on the main `bdk_wallet` library.
+- [`example_bitcoind_rpc_polling`](./example-crates/example_bitcoind_rpc_polling): A command line Bitcoin wallet application built on top of `example_cli` and the `bitcoind_rpc` crate. It shows the power of the bdk tools (`chain` + `file_store` + `bitcoind_rpc`), without depending on the main `bdk_wallet` library.
 - [`wallet_esplora_blocking`](./example-crates/wallet_esplora_blocking): Uses the `Wallet` to sync and spend using the Esplora blocking interface.
 - [`wallet_esplora_async`](./example-crates/wallet_esplora_async): Uses the `Wallet` to sync and spend using the Esplora asynchronous interface.
 - [`wallet_electrum`](./example-crates/wallet_electrum): Uses the `Wallet` to sync and spend using Electrum.
@@ -67,14 +68,12 @@ This library should compile with any combination of features with Rust 1.63.0.
 To build with the MSRV you will need to pin dependencies as follows:
 
 ```shell
-# zip 0.6.3 has MSRV 1.64.0
-cargo update -p zip --precise "0.6.2"
-# time 0.3.21 has MSRV 1.65.0
+cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
 cargo update -p time --precise "0.3.20"
-# jobserver 0.1.27 has MSRV 1.66.0
-cargo update -p jobserver --precise "0.1.26"
-# home 0.5.9 has MSRV 1.70.0
 cargo update -p home --precise "0.5.5"
+cargo update -p proptest --precise "1.2.0"
+cargo update -p url --precise "2.5.0"
+cargo update -p cc --precise "1.0.105"
 ```
 
 ## License

clippy.toml

@@ -1 +1 @@
-msrv="1.63.0"
+msrv="1.63.0"

crates/bdk/examples/compiler.rs

@@ -1,66 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-extern crate bdk;
-extern crate bitcoin;
-extern crate miniscript;
-extern crate serde_json;
-
-use std::error::Error;
-use std::str::FromStr;
-
-use bitcoin::Network;
-use miniscript::policy::Concrete;
-use miniscript::Descriptor;
-
-use bdk::wallet::AddressIndex::New;
-use bdk::{KeychainKind, Wallet};
-
-/// Miniscript policy is a high level abstraction of spending conditions. Defined in the
-/// rust-miniscript library here  https://docs.rs/miniscript/7.0.0/miniscript/policy/index.html
-/// rust-miniscript provides a `compile()` function that can be used to compile any miniscript policy
-/// into a descriptor. This descriptor then in turn can be used in bdk a fully functioning wallet
-/// can be derived from the policy.
-///
-/// This example demonstrates the interaction between a bdk wallet and miniscript policy.
-
-fn main() -> Result<(), Box<dyn Error>> {
-    // We start with a generic miniscript policy string
-    let policy_str = "or(10@thresh(4,pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2)),1@and(older(4209713),thresh(2,pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),pk(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),pk(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068))))";
-    println!("Compiling policy: \n{}", policy_str);
-
-    // Parse the string as a [`Concrete`] type miniscript policy.
-    let policy = Concrete::<String>::from_str(policy_str)?;
-
-    // Create a `wsh` type descriptor from the policy.
-    // `policy.compile()` returns the resulting miniscript from the policy.
-    let descriptor = Descriptor::new_wsh(policy.compile()?)?;
-
-    println!("Compiled into following Descriptor: \n{}", descriptor);
-
-    // Create a new wallet from this descriptor
-    let mut wallet = Wallet::new_no_persist(&format!("{}", descriptor), None, Network::Regtest)?;
-
-    println!(
-        "First derived address from the descriptor: \n{}",
-        wallet.get_address(New)
-    );
-
-    // BDK also has it's own `Policy` structure to represent the spending condition in a more
-    // human readable json format.
-    let spending_policy = wallet.policies(KeychainKind::External)?;
-    println!(
-        "The BDK spending policy: \n{}",
-        serde_json::to_string_pretty(&spending_policy)?
-    );
-
-    Ok(())
-}

crates/bdk/src/types.rs

@@ -1,316 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-use alloc::boxed::Box;
-use core::convert::AsRef;
-use core::ops::Sub;
-
-use bdk_chain::ConfirmationTime;
-use bitcoin::blockdata::transaction::{OutPoint, Sequence, TxOut};
-use bitcoin::{psbt, Weight};
-
-use serde::{Deserialize, Serialize};
-
-/// Types of keychains
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
-pub enum KeychainKind {
-    /// External keychain, used for deriving recipient addresses.
-    External = 0,
-    /// Internal keychain, used for deriving change addresses.
-    Internal = 1,
-}
-
-impl KeychainKind {
-    /// Return [`KeychainKind`] as a byte
-    pub fn as_byte(&self) -> u8 {
-        match self {
-            KeychainKind::External => b'e',
-            KeychainKind::Internal => b'i',
-        }
-    }
-}
-
-impl AsRef<[u8]> for KeychainKind {
-    fn as_ref(&self) -> &[u8] {
-        match self {
-            KeychainKind::External => b"e",
-            KeychainKind::Internal => b"i",
-        }
-    }
-}
-
-/// Fee rate
-#[derive(Debug, Copy, Clone, PartialEq, PartialOrd)]
-// Internally stored as satoshi/vbyte
-pub struct FeeRate(f32);
-
-impl FeeRate {
-    /// Create a new instance checking the value provided
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    fn new_checked(value: f32) -> Self {
-        assert!(value.is_normal() || value == 0.0);
-        assert!(value.is_sign_positive());
-
-        FeeRate(value)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in sats/kwu
-    pub fn from_sat_per_kwu(sat_per_kwu: f32) -> Self {
-        FeeRate::new_checked(sat_per_kwu / 250.0_f32)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in sats/kvb
-    pub fn from_sat_per_kvb(sat_per_kvb: f32) -> Self {
-        FeeRate::new_checked(sat_per_kvb / 1000.0_f32)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in btc/kvbytes
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    pub fn from_btc_per_kvb(btc_per_kvb: f32) -> Self {
-        FeeRate::new_checked(btc_per_kvb * 1e5)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in satoshi/vbyte
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    pub fn from_sat_per_vb(sat_per_vb: f32) -> Self {
-        FeeRate::new_checked(sat_per_vb)
-    }
-
-    /// Create a new [`FeeRate`] with the default min relay fee value
-    pub const fn default_min_relay_fee() -> Self {
-        FeeRate(1.0)
-    }
-
-    /// Calculate fee rate from `fee` and weight units (`wu`).
-    pub fn from_wu(fee: u64, wu: Weight) -> FeeRate {
-        Self::from_vb(fee, wu.to_vbytes_ceil() as usize)
-    }
-
-    /// Calculate fee rate from `fee` and `vbytes`.
-    pub fn from_vb(fee: u64, vbytes: usize) -> FeeRate {
-        let rate = fee as f32 / vbytes as f32;
-        Self::from_sat_per_vb(rate)
-    }
-
-    /// Return the value as satoshi/vbyte
-    pub fn as_sat_per_vb(&self) -> f32 {
-        self.0
-    }
-
-    /// Return the value as satoshi/kwu
-    pub fn sat_per_kwu(&self) -> f32 {
-        self.0 * 250.0_f32
-    }
-
-    /// Calculate absolute fee in Satoshis using size in weight units.
-    pub fn fee_wu(&self, wu: Weight) -> u64 {
-        self.fee_vb(wu.to_vbytes_ceil() as usize)
-    }
-
-    /// Calculate absolute fee in Satoshis using size in virtual bytes.
-    pub fn fee_vb(&self, vbytes: usize) -> u64 {
-        (self.as_sat_per_vb() * vbytes as f32).ceil() as u64
-    }
-}
-
-impl Default for FeeRate {
-    fn default() -> Self {
-        FeeRate::default_min_relay_fee()
-    }
-}
-
-impl Sub for FeeRate {
-    type Output = Self;
-
-    fn sub(self, other: FeeRate) -> Self::Output {
-        FeeRate(self.0 - other.0)
-    }
-}
-
-/// Trait implemented by types that can be used to measure weight units.
-pub trait Vbytes {
-    /// Convert weight units to virtual bytes.
-    fn vbytes(self) -> usize;
-}
-
-impl Vbytes for usize {
-    fn vbytes(self) -> usize {
-        // ref: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki#transaction-size-calculations
-        (self as f32 / 4.0).ceil() as usize
-    }
-}
-
-/// An unspent output owned by a [`Wallet`].
-///
-/// [`Wallet`]: crate::Wallet
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
-pub struct LocalOutput {
-    /// Reference to a transaction output
-    pub outpoint: OutPoint,
-    /// Transaction output
-    pub txout: TxOut,
-    /// Type of keychain
-    pub keychain: KeychainKind,
-    /// Whether this UTXO is spent or not
-    pub is_spent: bool,
-    /// The derivation index for the script pubkey in the wallet
-    pub derivation_index: u32,
-    /// The confirmation time for transaction containing this utxo
-    pub confirmation_time: ConfirmationTime,
-}
-
-/// A [`Utxo`] with its `satisfaction_weight`.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct WeightedUtxo {
-    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
-    /// properly maintain the feerate when adding this input to a transaction during coin selection.
-    ///
-    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
-    pub satisfaction_weight: usize,
-    /// The UTXO
-    pub utxo: Utxo,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-/// An unspent transaction output (UTXO).
-pub enum Utxo {
-    /// A UTXO owned by the local wallet.
-    Local(LocalOutput),
-    /// A UTXO owned by another wallet.
-    Foreign {
-        /// The location of the output.
-        outpoint: OutPoint,
-        /// The nSequence value to set for this input.
-        sequence: Option<Sequence>,
-        /// The information about the input we require to add it to a PSBT.
-        // Box it to stop the type being too big.
-        psbt_input: Box<psbt::Input>,
-    },
-}
-
-impl Utxo {
-    /// Get the location of the UTXO
-    pub fn outpoint(&self) -> OutPoint {
-        match &self {
-            Utxo::Local(local) => local.outpoint,
-            Utxo::Foreign { outpoint, .. } => *outpoint,
-        }
-    }
-
-    /// Get the `TxOut` of the UTXO
-    pub fn txout(&self) -> &TxOut {
-        match &self {
-            Utxo::Local(local) => &local.txout,
-            Utxo::Foreign {
-                outpoint,
-                psbt_input,
-                ..
-            } => {
-                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
-                    return &prev_tx.output[outpoint.vout as usize];
-                }
-
-                if let Some(txout) = &psbt_input.witness_utxo {
-                    return txout;
-                }
-
-                unreachable!("Foreign UTXOs will always have one of these set")
-            }
-        }
-    }
-
-    /// Get the sequence number if an explicit sequence number has to be set for this input.
-    pub fn sequence(&self) -> Option<Sequence> {
-        match self {
-            Utxo::Local(_) => None,
-            Utxo::Foreign { sequence, .. } => *sequence,
-        }
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn can_store_feerate_in_const() {
-        const _MIN_RELAY: FeeRate = FeeRate::default_min_relay_fee();
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_neg_zero() {
-        let _ = FeeRate::from_sat_per_vb(-0.0);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_neg_value() {
-        let _ = FeeRate::from_sat_per_vb(-5.0);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_nan() {
-        let _ = FeeRate::from_sat_per_vb(f32::NAN);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_inf() {
-        let _ = FeeRate::from_sat_per_vb(f32::INFINITY);
-    }
-
-    #[test]
-    fn test_valid_feerate_pos_zero() {
-        let _ = FeeRate::from_sat_per_vb(0.0);
-    }
-
-    #[test]
-    fn test_fee_from_btc_per_kvb() {
-        let fee = FeeRate::from_btc_per_kvb(1e-5);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_vbyte() {
-        let fee = FeeRate::from_sat_per_vb(1.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_default_min_relay_fee() {
-        let fee = FeeRate::default_min_relay_fee();
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_kvb() {
-        let fee = FeeRate::from_sat_per_kvb(1000.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_kwu() {
-        let fee = FeeRate::from_sat_per_kwu(250.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-        assert_eq!(fee.sat_per_kwu(), 250.0);
-    }
-}

crates/bdk/tests/common.rs

@@ -1,156 +0,0 @@
-#![allow(unused)]
-
-use bdk::{wallet::AddressIndex, KeychainKind, LocalOutput, Wallet};
-use bdk_chain::indexed_tx_graph::Indexer;
-use bdk_chain::{BlockId, ConfirmationTime};
-use bitcoin::hashes::Hash;
-use bitcoin::{Address, BlockHash, Network, OutPoint, Transaction, TxIn, TxOut, Txid};
-use std::str::FromStr;
-
-// Return a fake wallet that appears to be funded for testing.
-//
-// The funded wallet containing a tx with a 76_000 sats input and two outputs, one spending 25_000
-// to a foreign address and one returning 50_000 back to the wallet as change. The remaining 1000
-// sats are the transaction fee.
-pub fn get_funded_wallet_with_change(
-    descriptor: &str,
-    change: Option<&str>,
-) -> (Wallet, bitcoin::Txid) {
-    let mut wallet = Wallet::new_no_persist(descriptor, change, Network::Regtest).unwrap();
-    let change_address = wallet.get_address(AddressIndex::New).address;
-    let sendto_address = Address::from_str("bcrt1q3qtze4ys45tgdvguj66zrk4fu6hq3a3v9pfly5")
-        .expect("address")
-        .require_network(Network::Regtest)
-        .unwrap();
-
-    let tx0 = Transaction {
-        version: 1,
-        lock_time: bitcoin::absolute::LockTime::ZERO,
-        input: vec![TxIn {
-            previous_output: OutPoint {
-                txid: Txid::all_zeros(),
-                vout: 0,
-            },
-            script_sig: Default::default(),
-            sequence: Default::default(),
-            witness: Default::default(),
-        }],
-        output: vec![TxOut {
-            value: 76_000,
-            script_pubkey: change_address.script_pubkey(),
-        }],
-    };
-
-    let tx1 = Transaction {
-        version: 1,
-        lock_time: bitcoin::absolute::LockTime::ZERO,
-        input: vec![TxIn {
-            previous_output: OutPoint {
-                txid: tx0.txid(),
-                vout: 0,
-            },
-            script_sig: Default::default(),
-            sequence: Default::default(),
-            witness: Default::default(),
-        }],
-        output: vec![
-            TxOut {
-                value: 50_000,
-                script_pubkey: change_address.script_pubkey(),
-            },
-            TxOut {
-                value: 25_000,
-                script_pubkey: sendto_address.script_pubkey(),
-            },
-        ],
-    };
-
-    wallet
-        .insert_checkpoint(BlockId {
-            height: 1_000,
-            hash: BlockHash::all_zeros(),
-        })
-        .unwrap();
-    wallet
-        .insert_checkpoint(BlockId {
-            height: 2_000,
-            hash: BlockHash::all_zeros(),
-        })
-        .unwrap();
-    wallet
-        .insert_tx(
-            tx0,
-            ConfirmationTime::Confirmed {
-                height: 1_000,
-                time: 100,
-            },
-        )
-        .unwrap();
-    wallet
-        .insert_tx(
-            tx1.clone(),
-            ConfirmationTime::Confirmed {
-                height: 2_000,
-                time: 200,
-            },
-        )
-        .unwrap();
-
-    (wallet, tx1.txid())
-}
-
-// Return a fake wallet that appears to be funded for testing.
-//
-// The funded wallet containing a tx with a 76_000 sats input and two outputs, one spending 25_000
-// to a foreign address and one returning 50_000 back to the wallet as change. The remaining 1000
-// sats are the transaction fee.
-pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
-    get_funded_wallet_with_change(descriptor, None)
-}
-
-pub fn get_test_wpkh() -> &'static str {
-    "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)"
-}
-
-pub fn get_test_single_sig_csv() -> &'static str {
-    // and(pk(Alice),older(6))
-    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),older(6)))"
-}
-
-pub fn get_test_a_or_b_plus_csv() -> &'static str {
-    // or(pk(Alice),and(pk(Bob),older(144)))
-    "wsh(or_d(pk(cRjo6jqfVNP33HhSS76UhXETZsGTZYx8FMFvR9kpbtCSV1PmdZdu),and_v(v:pk(cMnkdebixpXMPfkcNEjjGin7s94hiehAH4mLbYkZoh9KSiNNmqC8),older(144))))"
-}
-
-pub fn get_test_single_sig_cltv() -> &'static str {
-    // and(pk(Alice),after(100000))
-    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100000)))"
-}
-
-pub fn get_test_tr_single_sig() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG)"
-}
-
-pub fn get_test_tr_with_taptree() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}
-
-pub fn get_test_tr_with_taptree_both_priv() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(cNaQCDwmmh4dS9LzCgVtyy1e1xjCJ21GUDHe9K98nzb689JvinGV)})"
-}
-
-pub fn get_test_tr_repeated_key() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100)),and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(200))})"
-}
-
-pub fn get_test_tr_single_sig_xprv() -> &'static str {
-    "tr(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*)"
-}
-
-pub fn get_test_tr_with_taptree_xprv() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}
-
-pub fn get_test_tr_dup_keys() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}

crates/bitcoind_rpc/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_bitcoind_rpc"
-version = "0.5.0"
+version = "0.13.0"
 edition = "2021"
 rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"
@@ -13,14 +13,12 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-# For no-std, remember to enable the bitcoin/no-std feature
-bitcoin = { version = "0.30", default-features = false }
-bitcoincore-rpc = { version = "0.17" }
-bdk_chain = { path = "../chain", version = "0.10", default-features = false }
+bitcoin = { version = "0.32.0", default-features = false }
+bitcoincore-rpc = { version = "0.19.0" }
+bdk_chain = { path = "../chain", version = "0.17", default-features = false }
 
 [dev-dependencies]
-bitcoind = { version = "0.33", features = ["25_0"] }
-anyhow = { version = "1" }
+bdk_testenv = { path = "../testenv", default-features = false }
 
 [features]
 default = ["std"]

crates/bitcoind_rpc/tests/test_emitter.rs

@@ -2,160 +2,14 @@ use std::collections::{BTreeMap, BTreeSet};
 
 use bdk_bitcoind_rpc::Emitter;
 use bdk_chain::{
-    bitcoin::{Address, Amount, BlockHash, Txid},
-    keychain::Balance,
-    local_chain::{self, CheckPoint, LocalChain},
-    Append, BlockId, IndexedTxGraph, SpkTxOutIndex,
+    bitcoin::{Address, Amount, Txid},
+    local_chain::{CheckPoint, LocalChain},
+    spk_txout::SpkTxOutIndex,
+    Balance, BlockId, IndexedTxGraph, Merge,
 };
-use bitcoin::{
-    address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash,
-    secp256k1::rand::random, Block, CompactTarget, OutPoint, ScriptBuf, ScriptHash, Transaction,
-    TxIn, TxOut, WScriptHash,
-};
-use bitcoincore_rpc::{
-    bitcoincore_rpc_json::{GetBlockTemplateModes, GetBlockTemplateRules},
-    RpcApi,
-};
-
-struct TestEnv {
-    #[allow(dead_code)]
-    daemon: bitcoind::BitcoinD,
-    client: bitcoincore_rpc::Client,
-}
-
-impl TestEnv {
-    fn new() -> anyhow::Result<Self> {
-        let daemon = match std::env::var_os("TEST_BITCOIND") {
-            Some(bitcoind_path) => bitcoind::BitcoinD::new(bitcoind_path),
-            None => bitcoind::BitcoinD::from_downloaded(),
-        }?;
-        let client = bitcoincore_rpc::Client::new(
-            &daemon.rpc_url(),
-            bitcoincore_rpc::Auth::CookieFile(daemon.params.cookie_file.clone()),
-        )?;
-        Ok(Self { daemon, client })
-    }
-
-    fn mine_blocks(
-        &self,
-        count: usize,
-        address: Option<Address>,
-    ) -> anyhow::Result<Vec<BlockHash>> {
-        let coinbase_address = match address {
-            Some(address) => address,
-            None => self.client.get_new_address(None, None)?.assume_checked(),
-        };
-        let block_hashes = self
-            .client
-            .generate_to_address(count as _, &coinbase_address)?;
-        Ok(block_hashes)
-    }
-
-    fn mine_empty_block(&self) -> anyhow::Result<(usize, BlockHash)> {
-        let bt = self.client.get_block_template(
-            GetBlockTemplateModes::Template,
-            &[GetBlockTemplateRules::SegWit],
-            &[],
-        )?;
-
-        let txdata = vec![Transaction {
-            version: 1,
-            lock_time: bitcoin::absolute::LockTime::from_height(0)?,
-            input: vec![TxIn {
-                previous_output: bitcoin::OutPoint::default(),
-                script_sig: ScriptBuf::builder()
-                    .push_int(bt.height as _)
-                    // randomn number so that re-mining creates unique block
-                    .push_int(random())
-                    .into_script(),
-                sequence: bitcoin::Sequence::default(),
-                witness: bitcoin::Witness::new(),
-            }],
-            output: vec![TxOut {
-                value: 0,
-                script_pubkey: ScriptBuf::new_p2sh(&ScriptHash::all_zeros()),
-            }],
-        }];
-
-        let bits: [u8; 4] = bt
-            .bits
-            .clone()
-            .try_into()
-            .expect("rpc provided us with invalid bits");
-
-        let mut block = Block {
-            header: Header {
-                version: bitcoin::block::Version::default(),
-                prev_blockhash: bt.previous_block_hash,
-                merkle_root: TxMerkleNode::all_zeros(),
-                time: Ord::max(bt.min_time, std::time::UNIX_EPOCH.elapsed()?.as_secs()) as u32,
-                bits: CompactTarget::from_consensus(u32::from_be_bytes(bits)),
-                nonce: 0,
-            },
-            txdata,
-        };
-
-        block.header.merkle_root = block.compute_merkle_root().expect("must compute");
-
-        for nonce in 0..=u32::MAX {
-            block.header.nonce = nonce;
-            if block.header.target().is_met_by(block.block_hash()) {
-                break;
-            }
-        }
-
-        self.client.submit_block(&block)?;
-        Ok((bt.height as usize, block.block_hash()))
-    }
-
-    fn invalidate_blocks(&self, count: usize) -> anyhow::Result<()> {
-        let mut hash = self.client.get_best_block_hash()?;
-        for _ in 0..count {
-            let prev_hash = self.client.get_block_info(&hash)?.previousblockhash;
-            self.client.invalidate_block(&hash)?;
-            match prev_hash {
-                Some(prev_hash) => hash = prev_hash,
-                None => break,
-            }
-        }
-        Ok(())
-    }
-
-    fn reorg(&self, count: usize) -> anyhow::Result<Vec<BlockHash>> {
-        let start_height = self.client.get_block_count()?;
-        self.invalidate_blocks(count)?;
-
-        let res = self.mine_blocks(count, None);
-        assert_eq!(
-            self.client.get_block_count()?,
-            start_height,
-            "reorg should not result in height change"
-        );
-        res
-    }
-
-    fn reorg_empty_blocks(&self, count: usize) -> anyhow::Result<Vec<(usize, BlockHash)>> {
-        let start_height = self.client.get_block_count()?;
-        self.invalidate_blocks(count)?;
-
-        let res = (0..count)
-            .map(|_| self.mine_empty_block())
-            .collect::<Result<Vec<_>, _>>()?;
-        assert_eq!(
-            self.client.get_block_count()?,
-            start_height,
-            "reorg should not result in height change"
-        );
-        Ok(res)
-    }
-
-    fn send(&self, address: &Address<NetworkChecked>, amount: Amount) -> anyhow::Result<Txid> {
-        let txid = self
-            .client
-            .send_to_address(address, amount, None, None, None, None, None, None)?;
-        Ok(txid)
-    }
-}
+use bdk_testenv::{anyhow, TestEnv};
+use bitcoin::{hashes::Hash, Block, OutPoint, ScriptBuf, WScriptHash};
+use bitcoincore_rpc::RpcApi;
 
 /// Ensure that blocks are emitted in order even after reorg.
 ///
@@ -166,17 +20,22 @@ impl TestEnv {
 #[test]
 pub fn test_sync_local_chain() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
-    let (mut local_chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
-    let mut emitter = Emitter::new(&env.client, local_chain.tip(), 0);
+    let network_tip = env.rpc_client().get_block_count()?;
+    let (mut local_chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
+    let mut emitter = Emitter::new(env.rpc_client(), local_chain.tip(), 0);
 
-    // mine some blocks and returned the actual block hashes
+    // Mine some blocks and return the actual block hashes.
+    // Because initializing `ElectrsD` already mines some blocks, we must include those too when
+    // returning block hashes.
     let exp_hashes = {
-        let mut hashes = vec![env.client.get_block_hash(0)?]; // include genesis block
-        hashes.extend(env.mine_blocks(101, None)?);
+        let mut hashes = (0..=network_tip)
+            .map(|height| env.rpc_client().get_block_hash(height))
+            .collect::<Result<Vec<_>, _>>()?;
+        hashes.extend(env.mine_blocks(101 - network_tip as usize, None)?);
         hashes
     };
 
-    // see if the emitter outputs the right blocks
+    // See if the emitter outputs the right blocks.
     println!("first sync:");
     while let Some(emission) = emitter.next_block()? {
         let height = emission.block_height();
@@ -188,26 +47,26 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
         );
 
         assert_eq!(
-            local_chain.apply_update(local_chain::Update {
-                tip: emission.checkpoint,
-                introduce_older_blocks: false,
-            })?,
-            BTreeMap::from([(height, Some(hash))]),
+            local_chain.apply_update(emission.checkpoint,)?,
+            [(height, Some(hash))].into(),
             "chain update changeset is unexpected",
         );
     }
 
     assert_eq!(
-        local_chain.blocks(),
-        &exp_hashes
+        local_chain
+            .iter_checkpoints()
+            .map(|cp| (cp.height(), cp.hash()))
+            .collect::<BTreeSet<_>>(),
+        exp_hashes
             .iter()
             .enumerate()
             .map(|(i, hash)| (i as u32, *hash))
-            .collect(),
+            .collect::<BTreeSet<_>>(),
         "final local_chain state is unexpected",
     );
 
-    // perform reorg
+    // Perform reorg.
     let reorged_blocks = env.reorg(6)?;
     let exp_hashes = exp_hashes
         .iter()
@@ -216,7 +75,7 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
         .cloned()
         .collect::<Vec<_>>();
 
-    // see if the emitter outputs the right blocks
+    // See if the emitter outputs the right blocks.
     println!("after reorg:");
     let mut exp_height = exp_hashes.len() - reorged_blocks.len();
     while let Some(emission) = emitter.next_block()? {
@@ -233,16 +92,15 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
         );
 
         assert_eq!(
-            local_chain.apply_update(local_chain::Update {
-                tip: emission.checkpoint,
-                introduce_older_blocks: false,
-            })?,
+            local_chain.apply_update(emission.checkpoint,)?,
             if exp_height == exp_hashes.len() - reorged_blocks.len() {
-                core::iter::once((height, Some(hash)))
-                    .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
-                    .collect::<bdk_chain::local_chain::ChangeSet>()
+                bdk_chain::local_chain::ChangeSet {
+                    blocks: core::iter::once((height, Some(hash)))
+                        .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
+                        .collect(),
+                }
             } else {
-                BTreeMap::from([(height, Some(hash))])
+                [(height, Some(hash))].into()
             },
             "chain update changeset is unexpected",
         );
@@ -251,12 +109,15 @@ pub fn test_sync_local_chain() -> anyhow::Result<()> {
     }
 
     assert_eq!(
-        local_chain.blocks(),
-        &exp_hashes
+        local_chain
+            .iter_checkpoints()
+            .map(|cp| (cp.height(), cp.hash()))
+            .collect::<BTreeSet<_>>(),
+        exp_hashes
             .iter()
             .enumerate()
             .map(|(i, hash)| (i as u32, *hash))
-            .collect(),
+            .collect::<BTreeSet<_>>(),
         "final local_chain state is unexpected after reorg",
     );
 
@@ -272,16 +133,25 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
 
     println!("getting new addresses!");
-    let addr_0 = env.client.get_new_address(None, None)?.assume_checked();
-    let addr_1 = env.client.get_new_address(None, None)?.assume_checked();
-    let addr_2 = env.client.get_new_address(None, None)?.assume_checked();
+    let addr_0 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let addr_1 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let addr_2 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
     println!("got new addresses!");
 
     println!("mining block!");
     env.mine_blocks(101, None)?;
     println!("mined blocks!");
 
-    let (mut chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
+    let (mut chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
     let mut indexed_tx_graph = IndexedTxGraph::<BlockId, _>::new({
         let mut index = SpkTxOutIndex::<usize>::default();
         index.insert_spk(0, addr_0.script_pubkey());
@@ -290,14 +160,11 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
         index
     });
 
-    let emitter = &mut Emitter::new(&env.client, chain.tip(), 0);
+    let emitter = &mut Emitter::new(env.rpc_client(), chain.tip(), 0);
 
     while let Some(emission) = emitter.next_block()? {
         let height = emission.block_height();
-        let _ = chain.apply_update(local_chain::Update {
-            tip: emission.checkpoint,
-            introduce_older_blocks: false,
-        })?;
+        let _ = chain.apply_update(emission.checkpoint)?;
         let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
         assert!(indexed_additions.is_empty());
     }
@@ -306,7 +173,7 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
     let exp_txids = {
         let mut txids = BTreeSet::new();
         for _ in 0..3 {
-            txids.insert(env.client.send_to_address(
+            txids.insert(env.rpc_client().send_to_address(
                 &addr_0,
                 Amount::from_sat(10_000),
                 None,
@@ -329,20 +196,20 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
         let indexed_additions = indexed_tx_graph.batch_insert_unconfirmed(mempool_txs);
         assert_eq!(
             indexed_additions
-                .graph
+                .tx_graph
                 .txs
                 .iter()
-                .map(|tx| tx.txid())
+                .map(|tx| tx.compute_txid())
                 .collect::<BTreeSet<Txid>>(),
             exp_txids,
             "changeset should have the 3 mempool transactions",
         );
-        assert!(indexed_additions.graph.anchors.is_empty());
+        assert!(indexed_additions.tx_graph.anchors.is_empty());
     }
 
     // mine a block that confirms the 3 txs
     let exp_block_hash = env.mine_blocks(1, None)?[0];
-    let exp_block_height = env.client.get_block_info(&exp_block_hash)?.height as u32;
+    let exp_block_height = env.rpc_client().get_block_info(&exp_block_hash)?.height as u32;
     let exp_anchors = exp_txids
         .iter()
         .map({
@@ -358,14 +225,11 @@ fn test_into_tx_graph() -> anyhow::Result<()> {
     {
         let emission = emitter.next_block()?.expect("must get mined block");
         let height = emission.block_height();
-        let _ = chain.apply_update(local_chain::Update {
-            tip: emission.checkpoint,
-            introduce_older_blocks: false,
-        })?;
+        let _ = chain.apply_update(emission.checkpoint)?;
         let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
-        assert!(indexed_additions.graph.txs.is_empty());
-        assert!(indexed_additions.graph.txouts.is_empty());
-        assert_eq!(indexed_additions.graph.anchors, exp_anchors);
+        assert!(indexed_additions.tx_graph.txs.is_empty());
+        assert!(indexed_additions.tx_graph.txouts.is_empty());
+        assert_eq!(indexed_additions.tx_graph.anchors, exp_anchors);
     }
 
     Ok(())
@@ -386,10 +250,10 @@ fn ensure_block_emitted_after_reorg_is_at_reorg_height() -> anyhow::Result<()> {
 
     let env = TestEnv::new()?;
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         EMITTER_START_HEIGHT as _,
     );
@@ -420,8 +284,7 @@ fn process_block(
     block: Block,
     block_height: u32,
 ) -> anyhow::Result<()> {
-    recv_chain
-        .apply_update(CheckPoint::from_header(&block.header, block_height).into_update(false))?;
+    recv_chain.apply_update(CheckPoint::from_header(&block.header, block_height))?;
     let _ = recv_graph.apply_block(block, block_height);
     Ok(())
 }
@@ -463,21 +326,24 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
 
     let env = TestEnv::new()?;
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         0,
     );
 
     // setup addresses
-    let addr_to_mine = env.client.get_new_address(None, None)?.assume_checked();
-    let spk_to_track = ScriptBuf::new_v0_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_mine = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
     let addr_to_track = Address::from_script(&spk_to_track, bitcoin::Network::Regtest)?;
 
     // setup receiver
-    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.client.get_block_hash(0)?);
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
     let mut recv_graph = IndexedTxGraph::<BlockId, _>::new({
         let mut recv_index = SpkTxOutIndex::default();
         recv_index.insert_spk((), spk_to_track.clone());
@@ -493,7 +359,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
 
         // lock outputs that send to `addr_to_track`
         let outpoints_to_lock = env
-            .client
+            .rpc_client()
             .get_transaction(&txid, None)?
             .transaction()?
             .output
@@ -502,7 +368,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
             .filter(|(_, txo)| txo.script_pubkey == spk_to_track)
             .map(|(vout, _)| OutPoint::new(txid, vout as _))
             .collect::<Vec<_>>();
-        env.client.lock_unspent(&outpoints_to_lock)?;
+        env.rpc_client().lock_unspent(&outpoints_to_lock)?;
 
         let _ = env.mine_blocks(1, None)?;
     }
@@ -513,7 +379,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
     assert_eq!(
         get_balance(&recv_chain, &recv_graph)?,
         Balance {
-            confirmed: SEND_AMOUNT.to_sat() * ADDITIONAL_COUNT as u64,
+            confirmed: SEND_AMOUNT * ADDITIONAL_COUNT as u64,
             ..Balance::default()
         },
         "initial balance must be correct",
@@ -527,8 +393,7 @@ fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
         assert_eq!(
             get_balance(&recv_chain, &recv_graph)?,
             Balance {
-                confirmed: SEND_AMOUNT.to_sat() * (ADDITIONAL_COUNT - reorg_count) as u64,
-                trusted_pending: SEND_AMOUNT.to_sat() * reorg_count as u64,
+                confirmed: SEND_AMOUNT * (ADDITIONAL_COUNT - reorg_count) as u64,
                 ..Balance::default()
             },
             "reorg_count: {}",
@@ -551,16 +416,19 @@ fn mempool_avoids_re_emission() -> anyhow::Result<()> {
 
     let env = TestEnv::new()?;
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         0,
     );
 
     // mine blocks and sync up emitter
-    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
     env.mine_blocks(BLOCKS_TO_MINE, Some(addr.clone()))?;
     while emitter.next_header()?.is_some() {}
 
@@ -573,7 +441,7 @@ fn mempool_avoids_re_emission() -> anyhow::Result<()> {
     let emitted_txids = emitter
         .mempool()?
         .into_iter()
-        .map(|(tx, _)| tx.txid())
+        .map(|(tx, _)| tx.compute_txid())
         .collect::<BTreeSet<Txid>>();
     assert_eq!(
         emitted_txids, exp_txids,
@@ -613,16 +481,19 @@ fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()
 
     let env = TestEnv::new()?;
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         0,
     );
 
     // mine blocks to get initial balance, sync emitter up to tip
-    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
     env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
     while emitter.next_header()?.is_some() {}
 
@@ -639,7 +510,7 @@ fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()
         emitter
             .mempool()?
             .into_iter()
-            .map(|(tx, _)| tx.txid())
+            .map(|(tx, _)| tx.compute_txid())
             .collect::<BTreeSet<_>>(),
         tx_introductions.iter().map(|&(_, txid)| txid).collect(),
         "first mempool emission should include all txs",
@@ -648,7 +519,7 @@ fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()
         emitter
             .mempool()?
             .into_iter()
-            .map(|(tx, _)| tx.txid())
+            .map(|(tx, _)| tx.compute_txid())
             .collect::<BTreeSet<_>>(),
         tx_introductions.iter().map(|&(_, txid)| txid).collect(),
         "second mempool emission should still include all txs",
@@ -668,7 +539,7 @@ fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()
             let emitted_txids = emitter
                 .mempool()?
                 .into_iter()
-                .map(|(tx, _)| tx.txid())
+                .map(|(tx, _)| tx.compute_txid())
                 .collect::<BTreeSet<_>>();
             assert_eq!(
                 emitted_txids, exp_txids,
@@ -698,16 +569,19 @@ fn mempool_during_reorg() -> anyhow::Result<()> {
 
     let env = TestEnv::new()?;
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         0,
     );
 
     // mine blocks to get initial balance
-    let addr = env.client.get_new_address(None, None)?.assume_checked();
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
     env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
 
     // introduce mempool tx at each block extension
@@ -723,9 +597,9 @@ fn mempool_during_reorg() -> anyhow::Result<()> {
         emitter
             .mempool()?
             .into_iter()
-            .map(|(tx, _)| tx.txid())
+            .map(|(tx, _)| tx.compute_txid())
             .collect::<BTreeSet<_>>(),
-        env.client
+        env.rpc_client()
             .get_raw_mempool()?
             .into_iter()
             .collect::<BTreeSet<_>>(),
@@ -744,7 +618,7 @@ fn mempool_during_reorg() -> anyhow::Result<()> {
         // emission.
         // TODO: How can have have reorg logic in `TestEnv` NOT blacklast old blocks first?
         let tx_introductions = dbg!(env
-            .client
+            .rpc_client()
             .get_raw_mempool_verbose()?
             .into_iter()
             .map(|(txid, entry)| (txid, entry.height as usize))
@@ -760,7 +634,7 @@ fn mempool_during_reorg() -> anyhow::Result<()> {
             let mempool = emitter
                 .mempool()?
                 .into_iter()
-                .map(|(tx, _)| tx.txid())
+                .map(|(tx, _)| tx.compute_txid())
                 .collect::<BTreeSet<_>>();
             let exp_mempool = tx_introductions
                 .iter()
@@ -775,7 +649,7 @@ fn mempool_during_reorg() -> anyhow::Result<()> {
             let mempool = emitter
                 .mempool()?
                 .into_iter()
-                .map(|(tx, _)| tx.txid())
+                .map(|(tx, _)| tx.compute_txid())
                 .collect::<BTreeSet<_>>();
             let exp_mempool = tx_introductions
                 .iter()
@@ -821,10 +695,10 @@ fn no_agreement_point() -> anyhow::Result<()> {
 
     // start height is 99
     let mut emitter = Emitter::new(
-        &env.client,
+        env.rpc_client(),
         CheckPoint::new(BlockId {
             height: 0,
-            hash: env.client.get_block_hash(0)?,
+            hash: env.rpc_client().get_block_hash(0)?,
         }),
         (PREMINE_COUNT - 2) as u32,
     );
@@ -842,12 +716,12 @@ fn no_agreement_point() -> anyhow::Result<()> {
     let block_hash_100a = block_header_100a.block_hash();
 
     // get hash for block 101a
-    let block_hash_101a = env.client.get_block_hash(101)?;
+    let block_hash_101a = env.rpc_client().get_block_hash(101)?;
 
     // invalidate blocks 99a, 100a, 101a
-    env.client.invalidate_block(&block_hash_99a)?;
-    env.client.invalidate_block(&block_hash_100a)?;
-    env.client.invalidate_block(&block_hash_101a)?;
+    env.rpc_client().invalidate_block(&block_hash_99a)?;
+    env.rpc_client().invalidate_block(&block_hash_100a)?;
+    env.rpc_client().invalidate_block(&block_hash_101a)?;
 
     // mine new blocks 99b, 100b, 101b
     env.mine_blocks(3, None)?;

crates/chain/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_chain"
-version = "0.10.0"
+version = "0.17.0"
 edition = "2021"
 rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"
@@ -13,19 +13,23 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-# For no-std, remember to enable the bitcoin/no-std feature
-bitcoin = { version = "0.30.0", default-features = false }
-serde_crate = { package = "serde", version = "1", optional = true, features = ["derive"] }
+bitcoin = { version = "0.32.0", default-features = false }
+serde_crate = { package = "serde", version = "1", optional = true, features = ["derive", "rc"] }
 
 # Use hashbrown as a feature flag to have HashSet and HashMap from it.
-# note versions > 0.9.1 breaks ours 1.57.0 MSRV.
 hashbrown = { version = "0.9.1", optional = true, features = ["serde"] }
-miniscript = { version = "10.0.0", optional = true, default-features = false }
+miniscript = { version = "12.0.0", optional = true, default-features = false }
+
+# Feature dependencies
+rusqlite_crate = { package = "rusqlite", version = "0.31.0", features = ["bundled"], optional = true }
+serde_json = {version = "1", optional = true }
 
 [dev-dependencies]
 rand = "0.8"
+proptest = "1.2.0"
 
 [features]
-default = ["std"]
-std = ["bitcoin/std", "miniscript/std"]
-serde = ["serde_crate", "bitcoin/serde"]
+default = ["std", "miniscript"]
+std = ["bitcoin/std", "miniscript?/std"]
+serde = ["serde_crate", "bitcoin/serde", "miniscript?/serde"]
+rusqlite = ["std", "rusqlite_crate", "serde", "serde_json"]

crates/chain/src/balance.rs

@@ -0,0 +1,57 @@
+use bitcoin::Amount;
+
+/// Balance, differentiated into various categories.
+#[derive(Debug, PartialEq, Eq, Clone, Default)]
+#[cfg_attr(
+    feature = "serde",
+    derive(serde::Deserialize, serde::Serialize),
+    serde(crate = "serde_crate",)
+)]
+pub struct Balance {
+    /// All coinbase outputs not yet matured
+    pub immature: Amount,
+    /// Unconfirmed UTXOs generated by a wallet tx
+    pub trusted_pending: Amount,
+    /// Unconfirmed UTXOs received from an external wallet
+    pub untrusted_pending: Amount,
+    /// Confirmed and immediately spendable balance
+    pub confirmed: Amount,
+}
+
+impl Balance {
+    /// Get sum of trusted_pending and confirmed coins.
+    ///
+    /// This is the balance you can spend right now that shouldn't get cancelled via another party
+    /// double spending it.
+    pub fn trusted_spendable(&self) -> Amount {
+        self.confirmed + self.trusted_pending
+    }
+
+    /// Get the whole balance visible to the wallet.
+    pub fn total(&self) -> Amount {
+        self.confirmed + self.trusted_pending + self.untrusted_pending + self.immature
+    }
+}
+
+impl core::fmt::Display for Balance {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(
+            f,
+            "{{ immature: {}, trusted_pending: {}, untrusted_pending: {}, confirmed: {} }}",
+            self.immature, self.trusted_pending, self.untrusted_pending, self.confirmed
+        )
+    }
+}
+
+impl core::ops::Add for Balance {
+    type Output = Self;
+
+    fn add(self, other: Self) -> Self {
+        Self {
+            immature: self.immature + other.immature,
+            trusted_pending: self.trusted_pending + other.trusted_pending,
+            untrusted_pending: self.untrusted_pending + other.untrusted_pending,
+            confirmed: self.confirmed + other.confirmed,
+        }
+    }
+}

crates/chain/src/chain_data.rs

@@ -74,11 +74,11 @@ impl ConfirmationTime {
     }
 }
 
-impl From<ChainPosition<ConfirmationTimeHeightAnchor>> for ConfirmationTime {
-    fn from(observed_as: ChainPosition<ConfirmationTimeHeightAnchor>) -> Self {
+impl From<ChainPosition<ConfirmationBlockTime>> for ConfirmationTime {
+    fn from(observed_as: ChainPosition<ConfirmationBlockTime>) -> Self {
         match observed_as {
             ChainPosition::Confirmed(a) => Self::Confirmed {
-                height: a.confirmation_height,
+                height: a.block_id.height,
                 time: a.confirmation_time,
             },
             ChainPosition::Unconfirmed(last_seen) => Self::Unconfirmed { last_seen },
@@ -145,9 +145,7 @@ impl From<(&u32, &BlockHash)> for BlockId {
     }
 }
 
-/// An [`Anchor`] implementation that also records the exact confirmation height of the transaction.
-///
-/// Note that the confirmation block and the anchor block can be different here.
+/// An [`Anchor`] implementation that also records the exact confirmation time of the transaction.
 ///
 /// Refer to [`Anchor`] for more details.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
@@ -156,70 +154,27 @@ impl From<(&u32, &BlockHash)> for BlockId {
     derive(serde::Deserialize, serde::Serialize),
     serde(crate = "serde_crate")
 )]
-pub struct ConfirmationHeightAnchor {
-    /// The exact confirmation height of the transaction.
-    ///
-    /// It is assumed that this value is never larger than the height of the anchor block.
-    pub confirmation_height: u32,
+pub struct ConfirmationBlockTime {
     /// The anchor block.
-    pub anchor_block: BlockId,
-}
-
-impl Anchor for ConfirmationHeightAnchor {
-    fn anchor_block(&self) -> BlockId {
-        self.anchor_block
-    }
-
-    fn confirmation_height_upper_bound(&self) -> u32 {
-        self.confirmation_height
-    }
-}
-
-impl AnchorFromBlockPosition for ConfirmationHeightAnchor {
-    fn from_block_position(_block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
-        Self {
-            anchor_block: block_id,
-            confirmation_height: block_id.height,
-        }
-    }
-}
-
-/// An [`Anchor`] implementation that also records the exact confirmation time and height of the
-/// transaction.
-///
-/// Note that the confirmation block and the anchor block can be different here.
-///
-/// Refer to [`Anchor`] for more details.
-#[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(crate = "serde_crate")
-)]
-pub struct ConfirmationTimeHeightAnchor {
-    /// The confirmation height of the transaction being anchored.
-    pub confirmation_height: u32,
+    pub block_id: BlockId,
     /// The confirmation time of the transaction being anchored.
     pub confirmation_time: u64,
-    /// The anchor block.
-    pub anchor_block: BlockId,
 }
 
-impl Anchor for ConfirmationTimeHeightAnchor {
+impl Anchor for ConfirmationBlockTime {
     fn anchor_block(&self) -> BlockId {
-        self.anchor_block
+        self.block_id
     }
 
     fn confirmation_height_upper_bound(&self) -> u32 {
-        self.confirmation_height
+        self.block_id.height
     }
 }
 
-impl AnchorFromBlockPosition for ConfirmationTimeHeightAnchor {
+impl AnchorFromBlockPosition for ConfirmationBlockTime {
     fn from_block_position(block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
         Self {
-            anchor_block: block_id,
-            confirmation_height: block_id.height,
+            block_id,
             confirmation_time: block.header.time as _,
         }
     }
@@ -305,19 +260,19 @@ mod test {
 
     #[test]
     fn chain_position_ord() {
-        let unconf1 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(10);
-        let unconf2 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(20);
-        let conf1 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
-            confirmation_height: 9,
-            anchor_block: BlockId {
-                height: 20,
+        let unconf1 = ChainPosition::<ConfirmationBlockTime>::Unconfirmed(10);
+        let unconf2 = ChainPosition::<ConfirmationBlockTime>::Unconfirmed(20);
+        let conf1 = ChainPosition::Confirmed(ConfirmationBlockTime {
+            confirmation_time: 20,
+            block_id: BlockId {
+                height: 9,
                 ..Default::default()
             },
         });
-        let conf2 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
-            confirmation_height: 12,
-            anchor_block: BlockId {
-                height: 15,
+        let conf2 = ChainPosition::Confirmed(ConfirmationBlockTime {
+            confirmation_time: 15,
+            block_id: BlockId {
+                height: 12,
                 ..Default::default()
             },
         });

crates/chain/src/descriptor_ext.rs

@@ -1,10 +1,25 @@
 use crate::miniscript::{Descriptor, DescriptorPublicKey};
+use bitcoin::hashes::{hash_newtype, sha256, Hash};
+
+hash_newtype! {
+    /// Represents the unique ID of a descriptor.
+    ///
+    /// This is useful for having a fixed-length unique representation of a descriptor,
+    /// in particular, we use it to persist application state changes related to the
+    /// descriptor without having to re-write the whole descriptor each time.
+    ///
+    pub struct DescriptorId(pub sha256::Hash);
+}
 
 /// A trait to extend the functionality of a miniscript descriptor.
 pub trait DescriptorExt {
     /// Returns the minimum value (in satoshis) at which an output is broadcastable.
     /// Panics if the descriptor wildcard is hardened.
     fn dust_value(&self) -> u64;
+
+    /// Returns the descriptor ID, calculated as the sha256 hash of the spk derived from the
+    /// descriptor at index 0.
+    fn descriptor_id(&self) -> DescriptorId;
 }
 
 impl DescriptorExt for Descriptor<DescriptorPublicKey> {
@@ -12,7 +27,12 @@ impl DescriptorExt for Descriptor<DescriptorPublicKey> {
         self.at_derivation_index(0)
             .expect("descriptor can't have hardened derivation")
             .script_pubkey()
-            .dust_value()
+            .minimal_non_dust()
             .to_sat()
     }
+
+    fn descriptor_id(&self) -> DescriptorId {
+        let spk = self.at_derivation_index(0).unwrap().script_pubkey();
+        DescriptorId(sha256::Hash::hash(spk.as_bytes()))
+    }
 }

crates/chain/src/indexed_tx_graph.rs

@@ -1,12 +1,13 @@
 //! Contains the [`IndexedTxGraph`] and associated types. Refer to the
 //! [`IndexedTxGraph`] documentation for more.
+use core::fmt::Debug;
+
 use alloc::vec::Vec;
 use bitcoin::{Block, OutPoint, Transaction, TxOut, Txid};
 
 use crate::{
-    keychain,
     tx_graph::{self, TxGraph},
-    Anchor, AnchorFromBlockPosition, Append, BlockId,
+    Anchor, AnchorFromBlockPosition, BlockId, Indexer, Merge,
 };
 
 /// The [`IndexedTxGraph`] combines a [`TxGraph`] and an [`Indexer`] implementation.
@@ -48,27 +49,30 @@ impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I> {
     pub fn apply_changeset(&mut self, changeset: ChangeSet<A, I::ChangeSet>) {
         self.index.apply_changeset(changeset.indexer);
 
-        for tx in &changeset.graph.txs {
+        for tx in &changeset.tx_graph.txs {
             self.index.index_tx(tx);
         }
-        for (&outpoint, txout) in &changeset.graph.txouts {
+        for (&outpoint, txout) in &changeset.tx_graph.txouts {
             self.index.index_txout(outpoint, txout);
         }
 
-        self.graph.apply_changeset(changeset.graph);
+        self.graph.apply_changeset(changeset.tx_graph);
     }
 
     /// Determines the [`ChangeSet`] between `self` and an empty [`IndexedTxGraph`].
     pub fn initial_changeset(&self) -> ChangeSet<A, I::ChangeSet> {
         let graph = self.graph.initial_changeset();
         let indexer = self.index.initial_changeset();
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 }
 
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
 where
-    I::ChangeSet: Default + Append,
+    I::ChangeSet: Default + Merge,
 {
     fn index_tx_graph_changeset(
         &mut self,
@@ -76,10 +80,10 @@ where
     ) -> I::ChangeSet {
         let mut changeset = I::ChangeSet::default();
         for added_tx in &tx_graph_changeset.txs {
-            changeset.append(self.index.index_tx(added_tx));
+            changeset.merge(self.index.index_tx(added_tx));
         }
         for (&added_outpoint, added_txout) in &tx_graph_changeset.txouts {
-            changeset.append(self.index.index_txout(added_outpoint, added_txout));
+            changeset.merge(self.index.index_txout(added_outpoint, added_txout));
         }
         changeset
     }
@@ -90,21 +94,30 @@ where
     pub fn apply_update(&mut self, update: TxGraph<A>) -> ChangeSet<A, I::ChangeSet> {
         let graph = self.graph.apply_update(update);
         let indexer = self.index_tx_graph_changeset(&graph);
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 
     /// Insert a floating `txout` of given `outpoint`.
     pub fn insert_txout(&mut self, outpoint: OutPoint, txout: TxOut) -> ChangeSet<A, I::ChangeSet> {
         let graph = self.graph.insert_txout(outpoint, txout);
         let indexer = self.index_tx_graph_changeset(&graph);
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 
     /// Insert and index a transaction into the graph.
     pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet<A, I::ChangeSet> {
         let graph = self.graph.insert_tx(tx);
         let indexer = self.index_tx_graph_changeset(&graph);
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 
     /// Insert an `anchor` for a given transaction.
@@ -138,21 +151,24 @@ where
 
         let mut indexer = I::ChangeSet::default();
         for (tx, _) in &txs {
-            indexer.append(self.index.index_tx(tx));
+            indexer.merge(self.index.index_tx(tx));
         }
 
         let mut graph = tx_graph::ChangeSet::default();
         for (tx, anchors) in txs {
             if self.index.is_tx_relevant(tx) {
-                let txid = tx.txid();
-                graph.append(self.graph.insert_tx(tx.clone()));
+                let txid = tx.compute_txid();
+                graph.merge(self.graph.insert_tx(tx.clone()));
                 for anchor in anchors {
-                    graph.append(self.graph.insert_anchor(txid, anchor));
+                    graph.merge(self.graph.insert_anchor(txid, anchor));
                 }
             }
         }
 
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 
     /// Batch insert unconfirmed transactions, filtering out those that are irrelevant.
@@ -177,7 +193,7 @@ where
 
         let mut indexer = I::ChangeSet::default();
         for (tx, _) in &txs {
-            indexer.append(self.index.index_tx(tx));
+            indexer.merge(self.index.index_tx(tx));
         }
 
         let graph = self.graph.batch_insert_unconfirmed(
@@ -186,7 +202,10 @@ where
                 .map(|(tx, seen_at)| (tx.clone(), seen_at)),
         );
 
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 
     /// Batch insert unconfirmed transactions.
@@ -204,14 +223,17 @@ where
     ) -> ChangeSet<A, I::ChangeSet> {
         let graph = self.graph.batch_insert_unconfirmed(txs);
         let indexer = self.index_tx_graph_changeset(&graph);
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
     }
 }
 
 /// Methods are available if the anchor (`A`) implements [`AnchorFromBlockPosition`].
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
 where
-    I::ChangeSet: Default + Append,
+    I::ChangeSet: Default + Merge,
     A: AnchorFromBlockPosition,
 {
     /// Batch insert all transactions of the given `block` of `height`, filtering out those that are
@@ -233,14 +255,14 @@ where
         };
         let mut changeset = ChangeSet::<A, I::ChangeSet>::default();
         for (tx_pos, tx) in block.txdata.iter().enumerate() {
-            changeset.indexer.append(self.index.index_tx(tx));
+            changeset.indexer.merge(self.index.index_tx(tx));
             if self.index.is_tx_relevant(tx) {
-                let txid = tx.txid();
+                let txid = tx.compute_txid();
                 let anchor = A::from_block_position(block, block_id, tx_pos);
-                changeset.graph.append(self.graph.insert_tx(tx.clone()));
+                changeset.tx_graph.merge(self.graph.insert_tx(tx.clone()));
                 changeset
-                    .graph
-                    .append(self.graph.insert_anchor(txid, anchor));
+                    .tx_graph
+                    .merge(self.graph.insert_anchor(txid, anchor));
             }
         }
         changeset
@@ -262,11 +284,20 @@ where
         let mut graph = tx_graph::ChangeSet::default();
         for (tx_pos, tx) in block.txdata.iter().enumerate() {
             let anchor = A::from_block_position(&block, block_id, tx_pos);
-            graph.append(self.graph.insert_anchor(tx.txid(), anchor));
-            graph.append(self.graph.insert_tx(tx.clone()));
+            graph.merge(self.graph.insert_anchor(tx.compute_txid(), anchor));
+            graph.merge(self.graph.insert_tx(tx.clone()));
         }
         let indexer = self.index_tx_graph_changeset(&graph);
-        ChangeSet { graph, indexer }
+        ChangeSet {
+            tx_graph: graph,
+            indexer,
+        }
+    }
+}
+
+impl<A, I> AsRef<TxGraph<A>> for IndexedTxGraph<A, I> {
+    fn as_ref(&self) -> &TxGraph<A> {
+        &self.graph
     }
 }
 
@@ -286,7 +317,7 @@ where
 #[must_use]
 pub struct ChangeSet<A, IA> {
     /// [`TxGraph`] changeset.
-    pub graph: tx_graph::ChangeSet<A>,
+    pub tx_graph: tx_graph::ChangeSet<A>,
     /// [`Indexer`] changeset.
     pub indexer: IA,
 }
@@ -294,61 +325,38 @@ pub struct ChangeSet<A, IA> {
 impl<A, IA: Default> Default for ChangeSet<A, IA> {
     fn default() -> Self {
         Self {
-            graph: Default::default(),
+            tx_graph: Default::default(),
             indexer: Default::default(),
         }
     }
 }
 
-impl<A: Anchor, IA: Append> Append for ChangeSet<A, IA> {
-    fn append(&mut self, other: Self) {
-        self.graph.append(other.graph);
-        self.indexer.append(other.indexer);
+impl<A: Anchor, IA: Merge> Merge for ChangeSet<A, IA> {
+    fn merge(&mut self, other: Self) {
+        self.tx_graph.merge(other.tx_graph);
+        self.indexer.merge(other.indexer);
     }
 
     fn is_empty(&self) -> bool {
-        self.graph.is_empty() && self.indexer.is_empty()
+        self.tx_graph.is_empty() && self.indexer.is_empty()
     }
 }
 
 impl<A, IA: Default> From<tx_graph::ChangeSet<A>> for ChangeSet<A, IA> {
     fn from(graph: tx_graph::ChangeSet<A>) -> Self {
         Self {
-            graph,
+            tx_graph: graph,
             ..Default::default()
         }
     }
 }
 
-impl<A, K> From<keychain::ChangeSet<K>> for ChangeSet<A, keychain::ChangeSet<K>> {
-    fn from(indexer: keychain::ChangeSet<K>) -> Self {
+#[cfg(feature = "miniscript")]
+impl<A> From<crate::keychain_txout::ChangeSet> for ChangeSet<A, crate::keychain_txout::ChangeSet> {
+    fn from(indexer: crate::keychain_txout::ChangeSet) -> Self {
         Self {
-            graph: Default::default(),
+            tx_graph: Default::default(),
             indexer,
         }
     }
 }
-
-/// Utilities for indexing transaction data.
-///
-/// Types which implement this trait can be used to construct an [`IndexedTxGraph`].
-/// This trait's methods should rarely be called directly.
-pub trait Indexer {
-    /// The resultant "changeset" when new transaction data is indexed.
-    type ChangeSet;
-
-    /// Scan and index the given `outpoint` and `txout`.
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet;
-
-    /// Scans a transaction for relevant outpoints, which are stored and indexed internally.
-    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet;
-
-    /// Apply changeset to itself.
-    fn apply_changeset(&mut self, changeset: Self::ChangeSet);
-
-    /// Determines the [`ChangeSet`] between `self` and an empty [`Indexer`].
-    fn initial_changeset(&self) -> Self::ChangeSet;
-
-    /// Determines whether the transaction should be included in the index.
-    fn is_tx_relevant(&self, tx: &Transaction) -> bool;
-}

crates/chain/src/indexer.rs

@@ -0,0 +1,33 @@
+//! [`Indexer`] provides utilities for indexing transaction data.
+
+use bitcoin::{OutPoint, Transaction, TxOut};
+
+#[cfg(feature = "miniscript")]
+pub mod keychain_txout;
+pub mod spk_txout;
+
+/// Utilities for indexing transaction data.
+///
+/// Types which implement this trait can be used to construct an [`IndexedTxGraph`].
+/// This trait's methods should rarely be called directly.
+///
+/// [`IndexedTxGraph`]: crate::IndexedTxGraph
+pub trait Indexer {
+    /// The resultant "changeset" when new transaction data is indexed.
+    type ChangeSet;
+
+    /// Scan and index the given `outpoint` and `txout`.
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet;
+
+    /// Scans a transaction for relevant outpoints, which are stored and indexed internally.
+    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet;
+
+    /// Apply changeset to itself.
+    fn apply_changeset(&mut self, changeset: Self::ChangeSet);
+
+    /// Determines the [`ChangeSet`](Indexer::ChangeSet) between `self` and an empty [`Indexer`].
+    fn initial_changeset(&self) -> Self::ChangeSet;
+
+    /// Determines whether the transaction should be included in the index.
+    fn is_tx_relevant(&self, tx: &Transaction) -> bool;
+}

crates/chain/src/indexer/keychain_txout.rs

@@ -0,0 +1,881 @@
+//! [`KeychainTxOutIndex`] controls how script pubkeys are revealed for multiple keychains and
+//! indexes [`TxOut`]s with them.
+
+use crate::{
+    collections::*,
+    miniscript::{Descriptor, DescriptorPublicKey},
+    spk_iter::BIP32_MAX_INDEX,
+    spk_txout::SpkTxOutIndex,
+    DescriptorExt, DescriptorId, Indexed, Indexer, KeychainIndexed, SpkIterator,
+};
+use alloc::{borrow::ToOwned, vec::Vec};
+use bitcoin::{Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid};
+use core::{
+    fmt::Debug,
+    ops::{Bound, RangeBounds},
+};
+
+use crate::Merge;
+
+/// The default lookahead for a [`KeychainTxOutIndex`]
+pub const DEFAULT_LOOKAHEAD: u32 = 25;
+
+/// [`KeychainTxOutIndex`] controls how script pubkeys are revealed for multiple keychains, and
+/// indexes [`TxOut`]s with them.
+///
+/// A single keychain is a chain of script pubkeys derived from a single [`Descriptor`]. Keychains
+/// are identified using the `K` generic. Script pubkeys are identified by the keychain that they
+/// are derived from `K`, as well as the derivation index `u32`.
+///
+/// There is a strict 1-to-1 relationship between descriptors and keychains. Each keychain has one
+/// and only one descriptor and each descriptor has one and only one keychain. The
+/// [`insert_descriptor`] method will return an error if you try and violate this invariant. This
+/// rule is a proxy for a stronger rule: no two descriptors should produce the same script pubkey.
+/// Having two descriptors produce the same script pubkey should cause whichever keychain derives
+/// the script pubkey first to be the effective owner of it but you should not rely on this
+/// behaviour. ⚠ It is up you, the developer, not to violate this invariant.
+///
+/// # Revealed script pubkeys
+///
+/// Tracking how script pubkeys are revealed is useful for collecting chain data. For example, if
+/// the user has requested 5 script pubkeys (to receive money with), we only need to use those
+/// script pubkeys to scan for chain data.
+///
+/// Call [`reveal_to_target`] or [`reveal_next_spk`] to reveal more script pubkeys.
+/// Call [`revealed_keychain_spks`] or [`revealed_spks`] to iterate through revealed script pubkeys.
+///
+/// # Lookahead script pubkeys
+///
+/// When an user first recovers a wallet (i.e. from a recovery phrase and/or descriptor), we will
+/// NOT have knowledge of which script pubkeys are revealed. So when we index a transaction or
+/// txout (using [`index_tx`]/[`index_txout`]) we scan the txouts against script pubkeys derived
+/// above the last revealed index. These additionally-derived script pubkeys are called the
+/// lookahead.
+///
+/// The [`KeychainTxOutIndex`] is constructed with the `lookahead` and cannot be altered. See
+/// [`DEFAULT_LOOKAHEAD`] for the value used in the `Default` implementation. Use [`new`] to set a
+/// custom `lookahead`.
+///
+/// # Unbounded script pubkey iterator
+///
+/// For script-pubkey-based chain sources (such as Electrum/Esplora), an initial scan is best done
+/// by iterating though derived script pubkeys one by one and requesting transaction histories for
+/// each script pubkey. We will stop after x-number of script pubkeys have empty histories. An
+/// unbounded script pubkey iterator is useful to pass to such a chain source because it doesn't
+/// require holding a reference to the index.
+///
+/// Call [`unbounded_spk_iter`] to get an unbounded script pubkey iterator for a given keychain.
+/// Call [`all_unbounded_spk_iters`] to get unbounded script pubkey iterators for all keychains.
+///
+/// # Change sets
+///
+/// Methods that can update the last revealed index or add keychains will return [`ChangeSet`] to report
+/// these changes. This should be persisted for future recovery.
+///
+/// ## Synopsis
+///
+/// ```
+/// use bdk_chain::indexer::keychain_txout::KeychainTxOutIndex;
+/// # use bdk_chain::{ miniscript::{Descriptor, DescriptorPublicKey} };
+/// # use core::str::FromStr;
+///
+/// // imagine our service has internal and external addresses but also addresses for users
+/// #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd)]
+/// enum MyKeychain {
+///     External,
+///     Internal,
+///     MyAppUser {
+///         user_id: u32
+///     }
+/// }
+///
+/// let mut txout_index = KeychainTxOutIndex::<MyKeychain>::default();
+///
+/// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
+/// # let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
+/// # let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
+/// # let (descriptor_42, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/2/*)").unwrap();
+/// let _ = txout_index.insert_descriptor(MyKeychain::External, external_descriptor)?;
+/// let _ = txout_index.insert_descriptor(MyKeychain::Internal, internal_descriptor)?;
+/// let _ = txout_index.insert_descriptor(MyKeychain::MyAppUser { user_id: 42 }, descriptor_42)?;
+///
+/// let new_spk_for_user = txout_index.reveal_next_spk(MyKeychain::MyAppUser{ user_id: 42 });
+/// # Ok::<_, bdk_chain::indexer::keychain_txout::InsertDescriptorError<_>>(())
+/// ```
+///
+/// [`Ord`]: core::cmp::Ord
+/// [`SpkTxOutIndex`]: crate::spk_txout_index::SpkTxOutIndex
+/// [`Descriptor`]: crate::miniscript::Descriptor
+/// [`reveal_to_target`]: Self::reveal_to_target
+/// [`reveal_next_spk`]: Self::reveal_next_spk
+/// [`revealed_keychain_spks`]: Self::revealed_keychain_spks
+/// [`revealed_spks`]: Self::revealed_spks
+/// [`index_tx`]: Self::index_tx
+/// [`index_txout`]: Self::index_txout
+/// [`new`]: Self::new
+/// [`unbounded_spk_iter`]: Self::unbounded_spk_iter
+/// [`all_unbounded_spk_iters`]: Self::all_unbounded_spk_iters
+/// [`outpoints`]: Self::outpoints
+/// [`txouts`]: Self::txouts
+/// [`unused_spks`]: Self::unused_spks
+/// [`insert_descriptor`]: Self::insert_descriptor
+#[derive(Clone, Debug)]
+pub struct KeychainTxOutIndex<K> {
+    inner: SpkTxOutIndex<(K, u32)>,
+    keychain_to_descriptor_id: BTreeMap<K, DescriptorId>,
+    descriptor_id_to_keychain: HashMap<DescriptorId, K>,
+    descriptors: HashMap<DescriptorId, Descriptor<DescriptorPublicKey>>,
+    last_revealed: HashMap<DescriptorId, u32>,
+    lookahead: u32,
+}
+
+impl<K> Default for KeychainTxOutIndex<K> {
+    fn default() -> Self {
+        Self::new(DEFAULT_LOOKAHEAD)
+    }
+}
+
+impl<K: Clone + Ord + Debug> Indexer for KeychainTxOutIndex<K> {
+    type ChangeSet = ChangeSet;
+
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
+        let mut changeset = ChangeSet::default();
+        if let Some((keychain, index)) = self.inner.scan_txout(outpoint, txout).cloned() {
+            let did = self
+                .keychain_to_descriptor_id
+                .get(&keychain)
+                .expect("invariant");
+            if self.last_revealed.get(did) < Some(&index) {
+                self.last_revealed.insert(*did, index);
+                changeset.last_revealed.insert(*did, index);
+                self.replenish_inner_index(*did, &keychain, self.lookahead);
+            }
+        }
+        changeset
+    }
+
+    fn index_tx(&mut self, tx: &bitcoin::Transaction) -> Self::ChangeSet {
+        let mut changeset = ChangeSet::default();
+        let txid = tx.compute_txid();
+        for (op, txout) in tx.output.iter().enumerate() {
+            changeset.merge(self.index_txout(OutPoint::new(txid, op as u32), txout));
+        }
+        changeset
+    }
+
+    fn initial_changeset(&self) -> Self::ChangeSet {
+        ChangeSet {
+            last_revealed: self.last_revealed.clone().into_iter().collect(),
+        }
+    }
+
+    fn apply_changeset(&mut self, changeset: Self::ChangeSet) {
+        self.apply_changeset(changeset)
+    }
+
+    fn is_tx_relevant(&self, tx: &bitcoin::Transaction) -> bool {
+        self.inner.is_relevant(tx)
+    }
+}
+
+impl<K> KeychainTxOutIndex<K> {
+    /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`.
+    ///
+    /// The `lookahead` is the number of script pubkeys to derive and cache from the internal
+    /// descriptors over and above the last revealed script index. Without a lookahead the index
+    /// will miss outputs you own when processing transactions whose output script pubkeys lie
+    /// beyond the last revealed index. In certain situations, such as when performing an initial
+    /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index
+    /// of the last revealed script pubkey actually is.
+    ///
+    /// Refer to [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
+    pub fn new(lookahead: u32) -> Self {
+        Self {
+            inner: SpkTxOutIndex::default(),
+            keychain_to_descriptor_id: Default::default(),
+            descriptors: Default::default(),
+            descriptor_id_to_keychain: Default::default(),
+            last_revealed: Default::default(),
+            lookahead,
+        }
+    }
+}
+
+/// Methods that are *re-exposed* from the internal [`SpkTxOutIndex`].
+impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
+    /// Return a reference to the internal [`SpkTxOutIndex`].
+    ///
+    /// **WARNING**: The internal index will contain lookahead spks. Refer to
+    /// [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
+    pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
+        &self.inner
+    }
+
+    /// Get the set of indexed outpoints, corresponding to tracked keychains.
+    pub fn outpoints(&self) -> &BTreeSet<KeychainIndexed<K, OutPoint>> {
+        self.inner.outpoints()
+    }
+
+    /// Iterate over known txouts that spend to tracked script pubkeys.
+    pub fn txouts(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = KeychainIndexed<K, (OutPoint, &TxOut)>> + ExactSizeIterator
+    {
+        self.inner
+            .txouts()
+            .map(|(index, op, txout)| (index.clone(), (op, txout)))
+    }
+
+    /// Finds all txouts on a transaction that has previously been scanned and indexed.
+    pub fn txouts_in_tx(
+        &self,
+        txid: Txid,
+    ) -> impl DoubleEndedIterator<Item = KeychainIndexed<K, (OutPoint, &TxOut)>> {
+        self.inner
+            .txouts_in_tx(txid)
+            .map(|(index, op, txout)| (index.clone(), (op, txout)))
+    }
+
+    /// Return the [`TxOut`] of `outpoint` if it has been indexed, and if it corresponds to a
+    /// tracked keychain.
+    ///
+    /// The associated keychain and keychain index of the txout's spk is also returned.
+    ///
+    /// This calls [`SpkTxOutIndex::txout`] internally.
+    pub fn txout(&self, outpoint: OutPoint) -> Option<KeychainIndexed<K, &TxOut>> {
+        self.inner
+            .txout(outpoint)
+            .map(|(index, txout)| (index.clone(), txout))
+    }
+
+    /// Return the script that exists under the given `keychain`'s `index`.
+    ///
+    /// This calls [`SpkTxOutIndex::spk_at_index`] internally.
+    pub fn spk_at_index(&self, keychain: K, index: u32) -> Option<ScriptBuf> {
+        self.inner.spk_at_index(&(keychain.clone(), index))
+    }
+
+    /// Returns the keychain and keychain index associated with the spk.
+    ///
+    /// This calls [`SpkTxOutIndex::index_of_spk`] internally.
+    pub fn index_of_spk(&self, script: ScriptBuf) -> Option<&(K, u32)> {
+        self.inner.index_of_spk(script)
+    }
+
+    /// Returns whether the spk under the `keychain`'s `index` has been used.
+    ///
+    /// Here, "unused" means that after the script pubkey was stored in the index, the index has
+    /// never scanned a transaction output with it.
+    ///
+    /// This calls [`SpkTxOutIndex::is_used`] internally.
+    pub fn is_used(&self, keychain: K, index: u32) -> bool {
+        self.inner.is_used(&(keychain, index))
+    }
+
+    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output
+    /// with it.
+    ///
+    /// This only has an effect when the `index` had been added to `self` already and was unused.
+    ///
+    /// Returns whether the spk under the given `keychain` and `index` is successfully
+    /// marked as used. Returns false either when there is no descriptor under the given
+    /// keychain, or when the spk is already marked as used.
+    ///
+    /// This is useful when you want to reserve a script pubkey for something but don't want to add
+    /// the transaction output using it to the index yet. Other callers will consider `index` on
+    /// `keychain` used until you call [`unmark_used`].
+    ///
+    /// This calls [`SpkTxOutIndex::mark_used`] internally.
+    ///
+    /// [`unmark_used`]: Self::unmark_used
+    pub fn mark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.mark_used(&(keychain, index))
+    }
+
+    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
+    /// `unused`.
+    ///
+    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
+    /// effect.
+    ///
+    /// This calls [`SpkTxOutIndex::unmark_used`] internally.
+    ///
+    /// [`mark_used`]: Self::mark_used
+    pub fn unmark_used(&mut self, keychain: K, index: u32) -> bool {
+        self.inner.unmark_used(&(keychain, index))
+    }
+
+    /// Computes the total value transfer effect `tx` has on the script pubkeys belonging to the
+    /// keychains in `range`. Value is *sent* when a script pubkey in the `range` is on an input and
+    /// *received* when it is on an output. For `sent` to be computed correctly, the output being
+    /// spent must have already been scanned by the index. Calculating received just uses the
+    /// [`Transaction`] outputs directly, so it will be correct even if it has not been scanned.
+    pub fn sent_and_received(
+        &self,
+        tx: &Transaction,
+        range: impl RangeBounds<K>,
+    ) -> (Amount, Amount) {
+        self.inner
+            .sent_and_received(tx, self.map_to_inner_bounds(range))
+    }
+
+    /// Computes the net value that this transaction gives to the script pubkeys in the index and
+    /// *takes* from the transaction outputs in the index. Shorthand for calling
+    /// [`sent_and_received`] and subtracting sent from received.
+    ///
+    /// This calls [`SpkTxOutIndex::net_value`] internally.
+    ///
+    /// [`sent_and_received`]: Self::sent_and_received
+    pub fn net_value(&self, tx: &Transaction, range: impl RangeBounds<K>) -> SignedAmount {
+        self.inner.net_value(tx, self.map_to_inner_bounds(range))
+    }
+}
+
+impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
+    /// Return all keychains and their corresponding descriptors.
+    pub fn keychains(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = (K, &Descriptor<DescriptorPublicKey>)> + ExactSizeIterator + '_
+    {
+        self.keychain_to_descriptor_id
+            .iter()
+            .map(|(k, did)| (k.clone(), self.descriptors.get(did).expect("invariant")))
+    }
+
+    /// Insert a descriptor with a keychain associated to it.
+    ///
+    /// Adding a descriptor means you will be able to derive new script pubkeys under it and the
+    /// txout index will discover transaction outputs with those script pubkeys (once they've been
+    /// derived and added to the index).
+    ///
+    /// keychain <-> descriptor is a one-to-one mapping that cannot be changed. Attempting to do so
+    /// will return a [`InsertDescriptorError<K>`].
+    ///
+    /// [`KeychainTxOutIndex`] will prevent you from inserting two descriptors which derive the same
+    /// script pubkey at index 0, but it's up to you to ensure that descriptors don't collide at
+    /// other indices. If they do nothing catastrophic happens at the `KeychainTxOutIndex` level
+    /// (one keychain just becomes the defacto owner of that spk arbitrarily) but this may have
+    /// subtle implications up the application stack like one UTXO being missing from one keychain
+    /// because it has been assigned to another which produces the same script pubkey.
+    pub fn insert_descriptor(
+        &mut self,
+        keychain: K,
+        descriptor: Descriptor<DescriptorPublicKey>,
+    ) -> Result<bool, InsertDescriptorError<K>> {
+        let did = descriptor.descriptor_id();
+        if !self.keychain_to_descriptor_id.contains_key(&keychain)
+            && !self.descriptor_id_to_keychain.contains_key(&did)
+        {
+            self.descriptors.insert(did, descriptor.clone());
+            self.keychain_to_descriptor_id.insert(keychain.clone(), did);
+            self.descriptor_id_to_keychain.insert(did, keychain.clone());
+            self.replenish_inner_index(did, &keychain, self.lookahead);
+            return Ok(true);
+        }
+
+        if let Some(existing_desc_id) = self.keychain_to_descriptor_id.get(&keychain) {
+            let descriptor = self.descriptors.get(existing_desc_id).expect("invariant");
+            if *existing_desc_id != did {
+                return Err(InsertDescriptorError::KeychainAlreadyAssigned {
+                    existing_assignment: descriptor.clone(),
+                    keychain,
+                });
+            }
+        }
+
+        if let Some(existing_keychain) = self.descriptor_id_to_keychain.get(&did) {
+            let descriptor = self.descriptors.get(&did).expect("invariant").clone();
+
+            if *existing_keychain != keychain {
+                return Err(InsertDescriptorError::DescriptorAlreadyAssigned {
+                    existing_assignment: existing_keychain.clone(),
+                    descriptor,
+                });
+            }
+        }
+
+        Ok(false)
+    }
+
+    /// Gets the descriptor associated with the keychain. Returns `None` if the keychain doesn't
+    /// have a descriptor associated with it.
+    pub fn get_descriptor(&self, keychain: K) -> Option<&Descriptor<DescriptorPublicKey>> {
+        let did = self.keychain_to_descriptor_id.get(&keychain)?;
+        self.descriptors.get(did)
+    }
+
+    /// Get the lookahead setting.
+    ///
+    /// Refer to [`new`] for more information on the `lookahead`.
+    ///
+    /// [`new`]: Self::new
+    pub fn lookahead(&self) -> u32 {
+        self.lookahead
+    }
+
+    /// Store lookahead scripts until `target_index` (inclusive).
+    ///
+    /// This does not change the global `lookahead` setting.
+    pub fn lookahead_to_target(&mut self, keychain: K, target_index: u32) {
+        if let Some((next_index, _)) = self.next_index(keychain.clone()) {
+            let temp_lookahead = (target_index + 1)
+                .checked_sub(next_index)
+                .filter(|&index| index > 0);
+
+            if let Some(temp_lookahead) = temp_lookahead {
+                self.replenish_inner_index_keychain(keychain, temp_lookahead);
+            }
+        }
+    }
+
+    fn replenish_inner_index_did(&mut self, did: DescriptorId, lookahead: u32) {
+        if let Some(keychain) = self.descriptor_id_to_keychain.get(&did).cloned() {
+            self.replenish_inner_index(did, &keychain, lookahead);
+        }
+    }
+
+    fn replenish_inner_index_keychain(&mut self, keychain: K, lookahead: u32) {
+        if let Some(did) = self.keychain_to_descriptor_id.get(&keychain) {
+            self.replenish_inner_index(*did, &keychain, lookahead);
+        }
+    }
+
+    /// Syncs the state of the inner spk index after changes to a keychain
+    fn replenish_inner_index(&mut self, did: DescriptorId, keychain: &K, lookahead: u32) {
+        let descriptor = self.descriptors.get(&did).expect("invariant");
+        let next_store_index = self
+            .inner
+            .all_spks()
+            .range(&(keychain.clone(), u32::MIN)..=&(keychain.clone(), u32::MAX))
+            .last()
+            .map_or(0, |((_, index), _)| *index + 1);
+        let next_reveal_index = self.last_revealed.get(&did).map_or(0, |v| *v + 1);
+        for (new_index, new_spk) in
+            SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead)
+        {
+            let _inserted = self
+                .inner
+                .insert_spk((keychain.clone(), new_index), new_spk);
+            debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_store_index={}, next_reveal_index={}", keychain, lookahead, next_store_index, next_reveal_index);
+        }
+    }
+
+    /// Get an unbounded spk iterator over a given `keychain`. Returns `None` if the provided
+    /// keychain doesn't exist
+    pub fn unbounded_spk_iter(
+        &self,
+        keychain: K,
+    ) -> Option<SpkIterator<Descriptor<DescriptorPublicKey>>> {
+        let descriptor = self.get_descriptor(keychain)?.clone();
+        Some(SpkIterator::new(descriptor))
+    }
+
+    /// Get unbounded spk iterators for all keychains.
+    pub fn all_unbounded_spk_iters(
+        &self,
+    ) -> BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>> {
+        self.keychain_to_descriptor_id
+            .iter()
+            .map(|(k, did)| {
+                (
+                    k.clone(),
+                    SpkIterator::new(self.descriptors.get(did).expect("invariant").clone()),
+                )
+            })
+            .collect()
+    }
+
+    /// Iterate over revealed spks of keychains in `range`
+    pub fn revealed_spks(
+        &self,
+        range: impl RangeBounds<K>,
+    ) -> impl Iterator<Item = KeychainIndexed<K, ScriptBuf>> + '_ {
+        let start = range.start_bound();
+        let end = range.end_bound();
+        let mut iter_last_revealed = self
+            .keychain_to_descriptor_id
+            .range((start, end))
+            .map(|(k, did)| (k, self.last_revealed.get(did).cloned()));
+        let mut iter_spks = self
+            .inner
+            .all_spks()
+            .range(self.map_to_inner_bounds((start, end)));
+        let mut current_keychain = iter_last_revealed.next();
+        // The reason we need a tricky algorithm is because of the "lookahead" feature which means
+        // that some of the spks in the SpkTxoutIndex will not have been revealed yet. So we need to
+        // filter out those spks that are above the last_revealed for that keychain. To do this we
+        // iterate through the last_revealed for each keychain and the spks for each keychain in
+        // tandem. This minimizes BTreeMap queries.
+        core::iter::from_fn(move || loop {
+            let ((keychain, index), spk) = iter_spks.next()?;
+            // We need to find the last revealed that matches the current spk we are considering so
+            // we skip ahead.
+            while current_keychain?.0 < keychain {
+                current_keychain = iter_last_revealed.next();
+            }
+            let (current_keychain, last_revealed) = current_keychain?;
+
+            if current_keychain == keychain && Some(*index) <= last_revealed {
+                break Some(((keychain.clone(), *index), spk.clone()));
+            }
+        })
+    }
+
+    /// Iterate over revealed spks of the given `keychain` with ascending indices.
+    ///
+    /// This is a double ended iterator so you can easily reverse it to get an iterator where
+    /// the script pubkeys that were most recently revealed are first.
+    pub fn revealed_keychain_spks(
+        &self,
+        keychain: K,
+    ) -> impl DoubleEndedIterator<Item = Indexed<ScriptBuf>> + '_ {
+        let end = self
+            .last_revealed_index(keychain.clone())
+            .map(|v| v + 1)
+            .unwrap_or(0);
+        self.inner
+            .all_spks()
+            .range((keychain.clone(), 0)..(keychain.clone(), end))
+            .map(|((_, index), spk)| (*index, spk.clone()))
+    }
+
+    /// Iterate over revealed, but unused, spks of all keychains.
+    pub fn unused_spks(
+        &self,
+    ) -> impl DoubleEndedIterator<Item = KeychainIndexed<K, ScriptBuf>> + Clone + '_ {
+        self.keychain_to_descriptor_id.keys().flat_map(|keychain| {
+            self.unused_keychain_spks(keychain.clone())
+                .map(|(i, spk)| ((keychain.clone(), i), spk.clone()))
+        })
+    }
+
+    /// Iterate over revealed, but unused, spks of the given `keychain`.
+    /// Returns an empty iterator if the provided keychain doesn't exist.
+    pub fn unused_keychain_spks(
+        &self,
+        keychain: K,
+    ) -> impl DoubleEndedIterator<Item = Indexed<ScriptBuf>> + Clone + '_ {
+        let end = match self.keychain_to_descriptor_id.get(&keychain) {
+            Some(did) => self.last_revealed.get(did).map(|v| *v + 1).unwrap_or(0),
+            None => 0,
+        };
+
+        self.inner
+            .unused_spks((keychain.clone(), 0)..(keychain.clone(), end))
+            .map(|((_, i), spk)| (*i, spk))
+    }
+
+    /// Get the next derivation index for `keychain`. The next index is the index after the last revealed
+    /// derivation index.
+    ///
+    /// The second field in the returned tuple represents whether the next derivation index is new.
+    /// There are two scenarios where the next derivation index is reused (not new):
+    ///
+    /// 1. The keychain's descriptor has no wildcard, and a script has already been revealed.
+    /// 2. The number of revealed scripts has already reached 2^31 (refer to BIP-32).
+    ///
+    /// Not checking the second field of the tuple may result in address reuse.
+    ///
+    /// Returns None if the provided `keychain` doesn't exist.
+    pub fn next_index(&self, keychain: K) -> Option<(u32, bool)> {
+        let did = self.keychain_to_descriptor_id.get(&keychain)?;
+        let last_index = self.last_revealed.get(did).cloned();
+        let descriptor = self.descriptors.get(did).expect("invariant");
+
+        // we can only get the next index if the wildcard exists.
+        let has_wildcard = descriptor.has_wildcard();
+
+        Some(match last_index {
+            // if there is no index, next_index is always 0.
+            None => (0, true),
+            // descriptors without wildcards can only have one index.
+            Some(_) if !has_wildcard => (0, false),
+            // derivation index must be < 2^31 (BIP-32).
+            Some(index) if index > BIP32_MAX_INDEX => {
+                unreachable!("index is out of bounds")
+            }
+            Some(index) if index == BIP32_MAX_INDEX => (index, false),
+            // get the next derivation index.
+            Some(index) => (index + 1, true),
+        })
+    }
+
+    /// Get the last derivation index that is revealed for each keychain.
+    ///
+    /// Keychains with no revealed indices will not be included in the returned [`BTreeMap`].
+    pub fn last_revealed_indices(&self) -> BTreeMap<K, u32> {
+        self.last_revealed
+            .iter()
+            .filter_map(|(desc_id, index)| {
+                let keychain = self.descriptor_id_to_keychain.get(desc_id)?;
+                Some((keychain.clone(), *index))
+            })
+            .collect()
+    }
+
+    /// Get the last derivation index revealed for `keychain`. Returns None if the keychain doesn't
+    /// exist, or if the keychain doesn't have any revealed scripts.
+    pub fn last_revealed_index(&self, keychain: K) -> Option<u32> {
+        let descriptor_id = self.keychain_to_descriptor_id.get(&keychain)?;
+        self.last_revealed.get(descriptor_id).cloned()
+    }
+
+    /// Convenience method to call [`Self::reveal_to_target`] on multiple keychains.
+    pub fn reveal_to_target_multi(&mut self, keychains: &BTreeMap<K, u32>) -> ChangeSet {
+        let mut changeset = ChangeSet::default();
+
+        for (keychain, &index) in keychains {
+            if let Some((_, new_changeset)) = self.reveal_to_target(keychain.clone(), index) {
+                changeset.merge(new_changeset);
+            }
+        }
+
+        changeset
+    }
+
+    /// Reveals script pubkeys of the `keychain`'s descriptor **up to and including** the
+    /// `target_index`.
+    ///
+    /// If the `target_index` cannot be reached (due to the descriptor having no wildcard and/or
+    /// the `target_index` is in the hardened index range), this method will make a best-effort and
+    /// reveal up to the last possible index.
+    ///
+    /// This returns list of newly revealed indices (alongside their scripts) and a
+    /// [`ChangeSet`], which reports updates to the latest revealed index. If no new script
+    /// pubkeys are revealed, then both of these will be empty.
+    ///
+    /// Returns None if the provided `keychain` doesn't exist.
+    #[must_use]
+    pub fn reveal_to_target(
+        &mut self,
+        keychain: K,
+        target_index: u32,
+    ) -> Option<(Vec<Indexed<ScriptBuf>>, ChangeSet)> {
+        let mut changeset = ChangeSet::default();
+        let mut spks: Vec<Indexed<ScriptBuf>> = vec![];
+        while let Some((i, new)) = self.next_index(keychain.clone()) {
+            if !new || i > target_index {
+                break;
+            }
+            match self.reveal_next_spk(keychain.clone()) {
+                Some(((i, spk), change)) => {
+                    spks.push((i, spk));
+                    changeset.merge(change);
+                }
+                None => break,
+            }
+        }
+
+        Some((spks, changeset))
+    }
+
+    /// Attempts to reveal the next script pubkey for `keychain`.
+    ///
+    /// Returns the derivation index of the revealed script pubkey, the revealed script pubkey and a
+    /// [`ChangeSet`] which represents changes in the last revealed index (if any).
+    /// Returns None if the provided keychain doesn't exist.
+    ///
+    /// When a new script cannot be revealed, we return the last revealed script and an empty
+    /// [`ChangeSet`]. There are two scenarios when a new script pubkey cannot be derived:
+    ///
+    ///  1. The descriptor has no wildcard and already has one script revealed.
+    ///  2. The descriptor has already revealed scripts up to the numeric bound.
+    ///  3. There is no descriptor associated with the given keychain.
+    pub fn reveal_next_spk(&mut self, keychain: K) -> Option<(Indexed<ScriptBuf>, ChangeSet)> {
+        let (next_index, new) = self.next_index(keychain.clone())?;
+        let mut changeset = ChangeSet::default();
+
+        if new {
+            let did = self.keychain_to_descriptor_id.get(&keychain)?;
+            self.last_revealed.insert(*did, next_index);
+            changeset.last_revealed.insert(*did, next_index);
+            self.replenish_inner_index(*did, &keychain, self.lookahead);
+        }
+        let script = self
+            .inner
+            .spk_at_index(&(keychain.clone(), next_index))
+            .expect("we just inserted it");
+        Some(((next_index, script), changeset))
+    }
+
+    /// Gets the next unused script pubkey in the keychain. I.e., the script pubkey with the lowest
+    /// index that has not been used yet.
+    ///
+    /// This will derive and reveal a new script pubkey if no more unused script pubkeys exist.
+    ///
+    /// If the descriptor has no wildcard and already has a used script pubkey or if a descriptor
+    /// has used all scripts up to the derivation bounds, then the last derived script pubkey will be
+    /// returned.
+    ///
+    /// Returns `None` if there are no script pubkeys that have been used and no new script pubkey
+    /// could be revealed (see [`reveal_next_spk`] for when this happens).
+    ///
+    /// [`reveal_next_spk`]: Self::reveal_next_spk
+    pub fn next_unused_spk(&mut self, keychain: K) -> Option<(Indexed<ScriptBuf>, ChangeSet)> {
+        let next_unused = self
+            .unused_keychain_spks(keychain.clone())
+            .next()
+            .map(|(i, spk)| ((i, spk.to_owned()), ChangeSet::default()));
+
+        next_unused.or_else(|| self.reveal_next_spk(keychain))
+    }
+
+    /// Iterate over all [`OutPoint`]s that have `TxOut`s with script pubkeys derived from
+    /// `keychain`.
+    pub fn keychain_outpoints(
+        &self,
+        keychain: K,
+    ) -> impl DoubleEndedIterator<Item = Indexed<OutPoint>> + '_ {
+        self.keychain_outpoints_in_range(keychain.clone()..=keychain)
+            .map(|((_, i), op)| (i, op))
+    }
+
+    /// Iterate over [`OutPoint`]s that have script pubkeys derived from keychains in `range`.
+    pub fn keychain_outpoints_in_range<'a>(
+        &'a self,
+        range: impl RangeBounds<K> + 'a,
+    ) -> impl DoubleEndedIterator<Item = KeychainIndexed<K, OutPoint>> + 'a {
+        self.inner
+            .outputs_in_range(self.map_to_inner_bounds(range))
+            .map(|((k, i), op)| ((k.clone(), *i), op))
+    }
+
+    fn map_to_inner_bounds(&self, bound: impl RangeBounds<K>) -> impl RangeBounds<(K, u32)> {
+        let start = match bound.start_bound() {
+            Bound::Included(keychain) => Bound::Included((keychain.clone(), u32::MIN)),
+            Bound::Excluded(keychain) => Bound::Excluded((keychain.clone(), u32::MAX)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
+        let end = match bound.end_bound() {
+            Bound::Included(keychain) => Bound::Included((keychain.clone(), u32::MAX)),
+            Bound::Excluded(keychain) => Bound::Excluded((keychain.clone(), u32::MIN)),
+            Bound::Unbounded => Bound::Unbounded,
+        };
+
+        (start, end)
+    }
+
+    /// Returns the highest derivation index of the `keychain` where [`KeychainTxOutIndex`] has
+    /// found a [`TxOut`] with it's script pubkey.
+    pub fn last_used_index(&self, keychain: K) -> Option<u32> {
+        self.keychain_outpoints(keychain).last().map(|(i, _)| i)
+    }
+
+    /// Returns the highest derivation index of each keychain that [`KeychainTxOutIndex`] has found
+    /// a [`TxOut`] with it's script pubkey.
+    pub fn last_used_indices(&self) -> BTreeMap<K, u32> {
+        self.keychain_to_descriptor_id
+            .iter()
+            .filter_map(|(keychain, _)| {
+                self.last_used_index(keychain.clone())
+                    .map(|index| (keychain.clone(), index))
+            })
+            .collect()
+    }
+
+    /// Applies the `ChangeSet<K>` to the [`KeychainTxOutIndex<K>`]
+    pub fn apply_changeset(&mut self, changeset: ChangeSet) {
+        for (&desc_id, &index) in &changeset.last_revealed {
+            let v = self.last_revealed.entry(desc_id).or_default();
+            *v = index.max(*v);
+            self.replenish_inner_index_did(desc_id, self.lookahead);
+        }
+    }
+}
+
+#[derive(Clone, Debug, PartialEq)]
+/// Error returned from [`KeychainTxOutIndex::insert_descriptor`]
+pub enum InsertDescriptorError<K> {
+    /// The descriptor has already been assigned to a keychain so you can't assign it to another
+    DescriptorAlreadyAssigned {
+        /// The descriptor you have attempted to reassign
+        descriptor: Descriptor<DescriptorPublicKey>,
+        /// The keychain that the descriptor is already assigned to
+        existing_assignment: K,
+    },
+    /// The keychain is already assigned to a descriptor so you can't reassign it
+    KeychainAlreadyAssigned {
+        /// The keychain that you have attempted to reassign
+        keychain: K,
+        /// The descriptor that the keychain is already assigned to
+        existing_assignment: Descriptor<DescriptorPublicKey>,
+    },
+}
+
+impl<K: core::fmt::Debug> core::fmt::Display for InsertDescriptorError<K> {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self {
+            InsertDescriptorError::DescriptorAlreadyAssigned {
+                existing_assignment: existing,
+                descriptor,
+            } => {
+                write!(
+                    f,
+                    "attempt to re-assign descriptor {descriptor:?} already assigned to {existing:?}"
+                )
+            }
+            InsertDescriptorError::KeychainAlreadyAssigned {
+                existing_assignment: existing,
+                keychain,
+            } => {
+                write!(
+                    f,
+                    "attempt to re-assign keychain {keychain:?} already assigned to {existing:?}"
+                )
+            }
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl<K: core::fmt::Debug> std::error::Error for InsertDescriptorError<K> {}
+
+/// Represents updates to the derivation index of a [`KeychainTxOutIndex`].
+/// It maps each keychain `K` to a descriptor and its last revealed index.
+///
+/// It can be applied to [`KeychainTxOutIndex`] with [`apply_changeset`].
+///
+/// The `last_revealed` field is monotone in that [`merge`] will never decrease it.
+/// `keychains_added` is *not* monotone, once it is set any attempt to change it is subject to the
+/// same *one-to-one* keychain <-> descriptor mapping invariant as [`KeychainTxOutIndex`] itself.
+///
+/// [`KeychainTxOutIndex`]: crate::keychain_txout::KeychainTxOutIndex
+/// [`apply_changeset`]: crate::keychain_txout::KeychainTxOutIndex::apply_changeset
+/// [`merge`]: Self::merge
+#[derive(Clone, Debug, Default, PartialEq)]
+#[cfg_attr(
+    feature = "serde",
+    derive(serde::Deserialize, serde::Serialize),
+    serde(crate = "serde_crate")
+)]
+#[must_use]
+pub struct ChangeSet {
+    /// Contains for each descriptor_id the last revealed index of derivation
+    pub last_revealed: BTreeMap<DescriptorId, u32>,
+}
+
+impl Merge for ChangeSet {
+    /// Merge another [`ChangeSet`] into self.
+    fn merge(&mut self, other: Self) {
+        // for `last_revealed`, entries of `other` will take precedence ONLY if it is greater than
+        // what was originally in `self`.
+        for (desc_id, index) in other.last_revealed {
+            use crate::collections::btree_map::Entry;
+            match self.last_revealed.entry(desc_id) {
+                Entry::Vacant(entry) => {
+                    entry.insert(index);
+                }
+                Entry::Occupied(mut entry) => {
+                    if *entry.get() < index {
+                        entry.insert(index);
+                    }
+                }
+            }
+        }
+    }
+
+    /// Returns whether the changeset are empty.
+    fn is_empty(&self) -> bool {
+        self.last_revealed.is_empty()
+    }
+}

crates/chain/src/indexer/spk_txout.rs

@@ -1,10 +1,12 @@
+//! [`SpkTxOutIndex`] is an index storing [`TxOut`]s that have a script pubkey that matches those in a list.
+
 use core::ops::RangeBounds;
 
 use crate::{
     collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap},
-    indexed_tx_graph::Indexer,
+    Indexer,
 };
-use bitcoin::{self, OutPoint, Script, ScriptBuf, Transaction, TxOut, Txid};
+use bitcoin::{Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxOut, Txid};
 
 /// An index storing [`TxOut`]s that have a script pubkey that matches those in a list.
 ///
@@ -52,7 +54,7 @@ impl<I> Default for SpkTxOutIndex<I> {
     }
 }
 
-impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
+impl<I: Clone + Ord + core::fmt::Debug> Indexer for SpkTxOutIndex<I> {
     type ChangeSet = ();
 
     fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
@@ -76,7 +78,7 @@ impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
     }
 }
 
-impl<I: Clone + Ord> SpkTxOutIndex<I> {
+impl<I: Clone + Ord + core::fmt::Debug> SpkTxOutIndex<I> {
     /// Scans a transaction's outputs for matching script pubkeys.
     ///
     /// Typically, this is used in two situations:
@@ -86,7 +88,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// 2. When getting new data from the chain, you usually scan it before incorporating it into your chain state.
     pub fn scan(&mut self, tx: &Transaction) -> BTreeSet<I> {
         let mut scanned_indices = BTreeSet::new();
-        let txid = tx.txid();
+        let txid = tx.compute_txid();
         for (i, txout) in tx.output.iter().enumerate() {
             let op = OutPoint::new(txid, i as u32);
             if let Some(spk_i) = self.scan_txout(op, txout) {
@@ -174,8 +176,8 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Returns the script that has been inserted at the `index`.
     ///
     /// If that index hasn't been inserted yet, it will return `None`.
-    pub fn spk_at_index(&self, index: &I) -> Option<&Script> {
-        self.spks.get(index).map(|s| s.as_script())
+    pub fn spk_at_index(&self, index: &I) -> Option<ScriptBuf> {
+        self.spks.get(index).cloned()
     }
 
     /// The script pubkeys that are being tracked by the index.
@@ -206,7 +208,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// # Example
     ///
     /// ```rust
-    /// # use bdk_chain::SpkTxOutIndex;
+    /// # use bdk_chain::spk_txout::SpkTxOutIndex;
     ///
     /// // imagine our spks are indexed like (keychain, derivation_index).
     /// let txout_index = SpkTxOutIndex::<(u32, u32)>::default();
@@ -215,7 +217,10 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// let unused_change_spks =
     ///     txout_index.unused_spks((change_index, u32::MIN)..(change_index, u32::MAX));
     /// ```
-    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)> + Clone
+    pub fn unused_spks<R>(
+        &self,
+        range: R,
+    ) -> impl DoubleEndedIterator<Item = (&I, ScriptBuf)> + Clone + '_
     where
         R: RangeBounds<I>,
     {
@@ -229,7 +234,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Here, "unused" means that after the script pubkey was stored in the index, the index has
     /// never scanned a transaction output with it.
     pub fn is_used(&self, index: &I) -> bool {
-        self.unused.get(index).is_none()
+        !self.unused.contains(index)
     }
 
     /// Marks the script pubkey at `index` as used even though it hasn't seen an output spending to it.
@@ -266,41 +271,49 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     }
 
     /// Returns the index associated with the script pubkey.
-    pub fn index_of_spk(&self, script: &Script) -> Option<&I> {
-        self.spk_indices.get(script)
+    pub fn index_of_spk(&self, script: ScriptBuf) -> Option<&I> {
+        self.spk_indices.get(script.as_script())
     }
 
-    /// Computes total input value going from script pubkeys in the index (sent) and the total output
-    /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
-    /// correctly, the output being spent must have already been scanned by the index. Calculating
-    /// received just uses the [`Transaction`] outputs directly, so it will be correct even if it has
-    /// not been scanned.
-    pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
-        let mut sent = 0;
-        let mut received = 0;
+    /// Computes the total value transfer effect `tx` has on the script pubkeys in `range`. Value is
+    /// *sent* when a script pubkey in the `range` is on an input and *received* when it is on an
+    /// output. For `sent` to be computed correctly, the output being spent must have already been
+    /// scanned by the index. Calculating received just uses the [`Transaction`] outputs directly,
+    /// so it will be correct even if it has not been scanned.
+    pub fn sent_and_received(
+        &self,
+        tx: &Transaction,
+        range: impl RangeBounds<I>,
+    ) -> (Amount, Amount) {
+        let mut sent = Amount::ZERO;
+        let mut received = Amount::ZERO;
 
         for txin in &tx.input {
-            if let Some((_, txout)) = self.txout(txin.previous_output) {
-                sent += txout.value;
+            if let Some((index, txout)) = self.txout(txin.previous_output) {
+                if range.contains(index) {
+                    sent += txout.value;
+                }
             }
         }
         for txout in &tx.output {
-            if self.index_of_spk(&txout.script_pubkey).is_some() {
-                received += txout.value;
+            if let Some(index) = self.index_of_spk(txout.script_pubkey.clone()) {
+                if range.contains(index) {
+                    received += txout.value;
+                }
             }
         }
 
         (sent, received)
     }
 
-    /// Computes the net value that this transaction gives to the script pubkeys in the index and
-    /// *takes* from the transaction outputs in the index. Shorthand for calling
-    /// [`sent_and_received`] and subtracting sent from received.
+    /// Computes the net value transfer effect of `tx` on the script pubkeys in `range`. Shorthand
+    /// for calling [`sent_and_received`] and subtracting sent from received.
     ///
     /// [`sent_and_received`]: Self::sent_and_received
-    pub fn net_value(&self, tx: &Transaction) -> i64 {
-        let (sent, received) = self.sent_and_received(tx);
-        received as i64 - sent as i64
+    pub fn net_value(&self, tx: &Transaction, range: impl RangeBounds<I>) -> SignedAmount {
+        let (sent, received) = self.sent_and_received(tx, range);
+        received.to_signed().expect("valid `SignedAmount`")
+            - sent.to_signed().expect("valid `SignedAmount`")
     }
 
     /// Whether any of the inputs of this transaction spend a txout tracked or whether any output

crates/chain/src/keychain.rs

@@ -1,175 +0,0 @@
-//! Module for keychain related structures.
-//!
-//! A keychain here is a set of application-defined indexes for a miniscript descriptor where we can
-//! derive script pubkeys at a particular derivation index. The application's index is simply
-//! anything that implements `Ord`.
-//!
-//! [`KeychainTxOutIndex`] indexes script pubkeys of keychains and scans in relevant outpoints (that
-//! has a `txout` containing an indexed script pubkey). Internally, this uses [`SpkTxOutIndex`], but
-//! also maintains "revealed" and "lookahead" index counts per keychain.
-//!
-//! [`SpkTxOutIndex`]: crate::SpkTxOutIndex
-
-use crate::{collections::BTreeMap, Append};
-
-#[cfg(feature = "miniscript")]
-mod txout_index;
-#[cfg(feature = "miniscript")]
-pub use txout_index::*;
-
-/// Represents updates to the derivation index of a [`KeychainTxOutIndex`].
-/// It maps each keychain `K` to its last revealed index.
-///
-/// It can be applied to [`KeychainTxOutIndex`] with [`apply_changeset`]. [`ChangeSet`]s are
-/// monotone in that they will never decrease the revealed derivation index.
-///
-/// [`KeychainTxOutIndex`]: crate::keychain::KeychainTxOutIndex
-/// [`apply_changeset`]: crate::keychain::KeychainTxOutIndex::apply_changeset
-#[derive(Clone, Debug, PartialEq)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(
-        crate = "serde_crate",
-        bound(
-            deserialize = "K: Ord + serde::Deserialize<'de>",
-            serialize = "K: Ord + serde::Serialize"
-        )
-    )
-)]
-#[must_use]
-pub struct ChangeSet<K>(pub BTreeMap<K, u32>);
-
-impl<K> ChangeSet<K> {
-    /// Get the inner map of the keychain to its new derivation index.
-    pub fn as_inner(&self) -> &BTreeMap<K, u32> {
-        &self.0
-    }
-}
-
-impl<K: Ord> Append for ChangeSet<K> {
-    /// Append another [`ChangeSet`] into self.
-    ///
-    /// If the keychain already exists, increase the index when the other's index > self's index.
-    /// If the keychain did not exist, append the new keychain.
-    fn append(&mut self, mut other: Self) {
-        self.0.iter_mut().for_each(|(key, index)| {
-            if let Some(other_index) = other.0.remove(key) {
-                *index = other_index.max(*index);
-            }
-        });
-        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
-        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
-        self.0.extend(other.0);
-    }
-
-    /// Returns whether the changeset are empty.
-    fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-}
-
-impl<K> Default for ChangeSet<K> {
-    fn default() -> Self {
-        Self(Default::default())
-    }
-}
-
-impl<K> AsRef<BTreeMap<K, u32>> for ChangeSet<K> {
-    fn as_ref(&self) -> &BTreeMap<K, u32> {
-        &self.0
-    }
-}
-
-/// Balance, differentiated into various categories.
-#[derive(Debug, PartialEq, Eq, Clone, Default)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(crate = "serde_crate",)
-)]
-pub struct Balance {
-    /// All coinbase outputs not yet matured
-    pub immature: u64,
-    /// Unconfirmed UTXOs generated by a wallet tx
-    pub trusted_pending: u64,
-    /// Unconfirmed UTXOs received from an external wallet
-    pub untrusted_pending: u64,
-    /// Confirmed and immediately spendable balance
-    pub confirmed: u64,
-}
-
-impl Balance {
-    /// Get sum of trusted_pending and confirmed coins.
-    ///
-    /// This is the balance you can spend right now that shouldn't get cancelled via another party
-    /// double spending it.
-    pub fn trusted_spendable(&self) -> u64 {
-        self.confirmed + self.trusted_pending
-    }
-
-    /// Get the whole balance visible to the wallet.
-    pub fn total(&self) -> u64 {
-        self.confirmed + self.trusted_pending + self.untrusted_pending + self.immature
-    }
-}
-
-impl core::fmt::Display for Balance {
-    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-        write!(
-            f,
-            "{{ immature: {}, trusted_pending: {}, untrusted_pending: {}, confirmed: {} }}",
-            self.immature, self.trusted_pending, self.untrusted_pending, self.confirmed
-        )
-    }
-}
-
-impl core::ops::Add for Balance {
-    type Output = Self;
-
-    fn add(self, other: Self) -> Self {
-        Self {
-            immature: self.immature + other.immature,
-            trusted_pending: self.trusted_pending + other.trusted_pending,
-            untrusted_pending: self.untrusted_pending + other.untrusted_pending,
-            confirmed: self.confirmed + other.confirmed,
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn append_keychain_derivation_indices() {
-        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Debug)]
-        enum Keychain {
-            One,
-            Two,
-            Three,
-            Four,
-        }
-        let mut lhs_di = BTreeMap::<Keychain, u32>::default();
-        let mut rhs_di = BTreeMap::<Keychain, u32>::default();
-        lhs_di.insert(Keychain::One, 7);
-        lhs_di.insert(Keychain::Two, 0);
-        rhs_di.insert(Keychain::One, 3);
-        rhs_di.insert(Keychain::Two, 5);
-        lhs_di.insert(Keychain::Three, 3);
-        rhs_di.insert(Keychain::Four, 4);
-
-        let mut lhs = ChangeSet(lhs_di);
-        let rhs = ChangeSet(rhs_di);
-        lhs.append(rhs);
-
-        // Exiting index doesn't update if the new index in `other` is lower than `self`.
-        assert_eq!(lhs.0.get(&Keychain::One), Some(&7));
-        // Existing index updates if the new index in `other` is higher than `self`.
-        assert_eq!(lhs.0.get(&Keychain::Two), Some(&5));
-        // Existing index is unchanged if keychain doesn't exist in `other`.
-        assert_eq!(lhs.0.get(&Keychain::Three), Some(&3));
-        // New keychain gets added if the keychain is in `other` but not in `self`.
-        assert_eq!(lhs.0.get(&Keychain::Four), Some(&4));
-    }
-}

crates/chain/src/keychain/txout_index.rs

@@ -1,672 +0,0 @@
-use crate::{
-    collections::*,
-    indexed_tx_graph::Indexer,
-    miniscript::{Descriptor, DescriptorPublicKey},
-    spk_iter::BIP32_MAX_INDEX,
-    SpkIterator, SpkTxOutIndex,
-};
-use bitcoin::{OutPoint, Script, Transaction, TxOut, Txid};
-use core::{
-    fmt::Debug,
-    ops::{Bound, RangeBounds},
-};
-
-use crate::Append;
-
-const DEFAULT_LOOKAHEAD: u32 = 25;
-
-/// [`KeychainTxOutIndex`] controls how script pubkeys are revealed for multiple keychains, and
-/// indexes [`TxOut`]s with them.
-///
-/// A single keychain is a chain of script pubkeys derived from a single [`Descriptor`]. Keychains
-/// are identified using the `K` generic. Script pubkeys are identified by the keychain that they
-/// are derived from `K`, as well as the derivation index `u32`.
-///
-/// # Revealed script pubkeys
-///
-/// Tracking how script pubkeys are revealed is useful for collecting chain data. For example, if
-/// the user has requested 5 script pubkeys (to receive money with), we only need to use those
-/// script pubkeys to scan for chain data.
-///
-/// Call [`reveal_to_target`] or [`reveal_next_spk`] to reveal more script pubkeys.
-/// Call [`revealed_keychain_spks`] or [`revealed_spks`] to iterate through revealed script pubkeys.
-///
-/// # Lookahead script pubkeys
-///
-/// When an user first recovers a wallet (i.e. from a recovery phrase and/or descriptor), we will
-/// NOT have knowledge of which script pubkeys are revealed. So when we index a transaction or
-/// txout (using [`index_tx`]/[`index_txout`]) we scan the txouts against script pubkeys derived
-/// above the last revealed index. These additionally-derived script pubkeys are called the
-/// lookahead.
-///
-/// The [`KeychainTxOutIndex`] is constructed with the `lookahead` and cannot be altered. The
-/// default `lookahead` count is 1000. Use [`new`] to set a custom `lookahead`.
-///
-/// # Unbounded script pubkey iterator
-///
-/// For script-pubkey-based chain sources (such as Electrum/Esplora), an initial scan is best done
-/// by iterating though derived script pubkeys one by one and requesting transaction histories for
-/// each script pubkey. We will stop after x-number of script pubkeys have empty histories. An
-/// unbounded script pubkey iterator is useful to pass to such a chain source.
-///
-/// Call [`unbounded_spk_iter`] to get an unbounded script pubkey iterator for a given keychain.
-/// Call [`all_unbounded_spk_iters`] to get unbounded script pubkey iterators for all keychains.
-///
-/// # Change sets
-///
-/// Methods that can update the last revealed index will return [`super::ChangeSet`] to report
-/// these changes. This can be persisted for future recovery.
-///
-/// ## Synopsis
-///
-/// ```
-/// use bdk_chain::keychain::KeychainTxOutIndex;
-/// # use bdk_chain::{ miniscript::{Descriptor, DescriptorPublicKey} };
-/// # use core::str::FromStr;
-///
-/// // imagine our service has internal and external addresses but also addresses for users
-/// #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd)]
-/// enum MyKeychain {
-///     External,
-///     Internal,
-///     MyAppUser {
-///         user_id: u32
-///     }
-/// }
-///
-/// let mut txout_index = KeychainTxOutIndex::<MyKeychain>::default();
-///
-/// # let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
-/// # let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
-/// # let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
-/// # let (descriptor_for_user_42, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/2/*)").unwrap();
-/// txout_index.add_keychain(MyKeychain::External, external_descriptor);
-/// txout_index.add_keychain(MyKeychain::Internal, internal_descriptor);
-/// txout_index.add_keychain(MyKeychain::MyAppUser { user_id: 42 }, descriptor_for_user_42);
-///
-/// let new_spk_for_user = txout_index.reveal_next_spk(&MyKeychain::MyAppUser{ user_id: 42 });
-/// ```
-///
-/// [`Ord`]: core::cmp::Ord
-/// [`SpkTxOutIndex`]: crate::spk_txout_index::SpkTxOutIndex
-/// [`Descriptor`]: crate::miniscript::Descriptor
-/// [`reveal_to_target`]: KeychainTxOutIndex::reveal_to_target
-/// [`reveal_next_spk`]: KeychainTxOutIndex::reveal_next_spk
-/// [`revealed_keychain_spks`]: KeychainTxOutIndex::revealed_keychain_spks
-/// [`revealed_spks`]: KeychainTxOutIndex::revealed_spks
-/// [`index_tx`]: KeychainTxOutIndex::index_tx
-/// [`index_txout`]: KeychainTxOutIndex::index_txout
-/// [`new`]: KeychainTxOutIndex::new
-/// [`unbounded_spk_iter`]: KeychainTxOutIndex::unbounded_spk_iter
-/// [`all_unbounded_spk_iters`]: KeychainTxOutIndex::all_unbounded_spk_iters
-#[derive(Clone, Debug)]
-pub struct KeychainTxOutIndex<K> {
-    inner: SpkTxOutIndex<(K, u32)>,
-    // descriptors of each keychain
-    keychains: BTreeMap<K, Descriptor<DescriptorPublicKey>>,
-    // last revealed indexes
-    last_revealed: BTreeMap<K, u32>,
-    // lookahead settings for each keychain
-    lookahead: u32,
-}
-
-impl<K> Default for KeychainTxOutIndex<K> {
-    fn default() -> Self {
-        Self::new(DEFAULT_LOOKAHEAD)
-    }
-}
-
-impl<K: Clone + Ord + Debug> Indexer for KeychainTxOutIndex<K> {
-    type ChangeSet = super::ChangeSet<K>;
-
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
-        match self.inner.scan_txout(outpoint, txout).cloned() {
-            Some((keychain, index)) => self.reveal_to_target(&keychain, index).1,
-            None => super::ChangeSet::default(),
-        }
-    }
-
-    fn index_tx(&mut self, tx: &bitcoin::Transaction) -> Self::ChangeSet {
-        let mut changeset = super::ChangeSet::<K>::default();
-        for (op, txout) in tx.output.iter().enumerate() {
-            changeset.append(self.index_txout(OutPoint::new(tx.txid(), op as u32), txout));
-        }
-        changeset
-    }
-
-    fn initial_changeset(&self) -> Self::ChangeSet {
-        super::ChangeSet(self.last_revealed.clone())
-    }
-
-    fn apply_changeset(&mut self, changeset: Self::ChangeSet) {
-        self.apply_changeset(changeset)
-    }
-
-    fn is_tx_relevant(&self, tx: &bitcoin::Transaction) -> bool {
-        self.inner.is_relevant(tx)
-    }
-}
-
-impl<K> KeychainTxOutIndex<K> {
-    /// Construct a [`KeychainTxOutIndex`] with the given `lookahead`.
-    ///
-    /// The `lookahead` is the number of script pubkeys to derive and cache from the internal
-    /// descriptors over and above the last revealed script index. Without a lookahead the index
-    /// will miss outputs you own when processing transactions whose output script pubkeys lie
-    /// beyond the last revealed index. In certain situations, such as when performing an initial
-    /// scan of the blockchain during wallet import, it may be uncertain or unknown what the index
-    /// of the last revealed script pubkey actually is.
-    ///
-    /// Refer to [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
-    pub fn new(lookahead: u32) -> Self {
-        Self {
-            inner: SpkTxOutIndex::default(),
-            keychains: BTreeMap::new(),
-            last_revealed: BTreeMap::new(),
-            lookahead,
-        }
-    }
-}
-
-/// Methods that are *re-exposed* from the internal [`SpkTxOutIndex`].
-impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
-    /// Return a reference to the internal [`SpkTxOutIndex`].
-    ///
-    /// **WARNING:** The internal index will contain lookahead spks. Refer to
-    /// [struct-level docs](KeychainTxOutIndex) for more about `lookahead`.
-    pub fn inner(&self) -> &SpkTxOutIndex<(K, u32)> {
-        &self.inner
-    }
-
-    /// Get a reference to the set of indexed outpoints.
-    pub fn outpoints(&self) -> &BTreeSet<((K, u32), OutPoint)> {
-        self.inner.outpoints()
-    }
-
-    /// Iterate over known txouts that spend to tracked script pubkeys.
-    pub fn txouts(
-        &self,
-    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> + ExactSizeIterator {
-        self.inner
-            .txouts()
-            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
-    }
-
-    /// Finds all txouts on a transaction that has previously been scanned and indexed.
-    pub fn txouts_in_tx(
-        &self,
-        txid: Txid,
-    ) -> impl DoubleEndedIterator<Item = (K, u32, OutPoint, &TxOut)> {
-        self.inner
-            .txouts_in_tx(txid)
-            .map(|((k, i), op, txo)| (k.clone(), *i, op, txo))
-    }
-
-    /// Return the [`TxOut`] of `outpoint` if it has been indexed.
-    ///
-    /// The associated keychain and keychain index of the txout's spk is also returned.
-    ///
-    /// This calls [`SpkTxOutIndex::txout`] internally.
-    pub fn txout(&self, outpoint: OutPoint) -> Option<(K, u32, &TxOut)> {
-        self.inner
-            .txout(outpoint)
-            .map(|((k, i), txo)| (k.clone(), *i, txo))
-    }
-
-    /// Return the script that exists under the given `keychain`'s `index`.
-    ///
-    /// This calls [`SpkTxOutIndex::spk_at_index`] internally.
-    pub fn spk_at_index(&self, keychain: K, index: u32) -> Option<&Script> {
-        self.inner.spk_at_index(&(keychain, index))
-    }
-
-    /// Returns the keychain and keychain index associated with the spk.
-    ///
-    /// This calls [`SpkTxOutIndex::index_of_spk`] internally.
-    pub fn index_of_spk(&self, script: &Script) -> Option<(K, u32)> {
-        self.inner.index_of_spk(script).cloned()
-    }
-
-    /// Returns whether the spk under the `keychain`'s `index` has been used.
-    ///
-    /// Here, "unused" means that after the script pubkey was stored in the index, the index has
-    /// never scanned a transaction output with it.
-    ///
-    /// This calls [`SpkTxOutIndex::is_used`] internally.
-    pub fn is_used(&self, keychain: K, index: u32) -> bool {
-        self.inner.is_used(&(keychain, index))
-    }
-
-    /// Marks the script pubkey at `index` as used even though the tracker hasn't seen an output
-    /// with it.
-    ///
-    /// This only has an effect when the `index` had been added to `self` already and was unused.
-    ///
-    /// Returns whether the `index` was initially present as `unused`.
-    ///
-    /// This is useful when you want to reserve a script pubkey for something but don't want to add
-    /// the transaction output using it to the index yet. Other callers will consider `index` on
-    /// `keychain` used until you call [`unmark_used`].
-    ///
-    /// This calls [`SpkTxOutIndex::mark_used`] internally.
-    ///
-    /// [`unmark_used`]: Self::unmark_used
-    pub fn mark_used(&mut self, keychain: K, index: u32) -> bool {
-        self.inner.mark_used(&(keychain, index))
-    }
-
-    /// Undoes the effect of [`mark_used`]. Returns whether the `index` is inserted back into
-    /// `unused`.
-    ///
-    /// Note that if `self` has scanned an output with this script pubkey, then this will have no
-    /// effect.
-    ///
-    /// This calls [`SpkTxOutIndex::unmark_used`] internally.
-    ///
-    /// [`mark_used`]: Self::mark_used
-    pub fn unmark_used(&mut self, keychain: K, index: u32) -> bool {
-        self.inner.unmark_used(&(keychain, index))
-    }
-
-    /// Computes total input value going from script pubkeys in the index (sent) and the total output
-    /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
-    /// correctly, the output being spent must have already been scanned by the index. Calculating
-    /// received just uses the [`Transaction`] outputs directly, so it will be correct even if it has
-    /// not been scanned.
-    ///
-    /// This calls [`SpkTxOutIndex::sent_and_received`] internally.
-    pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
-        self.inner.sent_and_received(tx)
-    }
-
-    /// Computes the net value that this transaction gives to the script pubkeys in the index and
-    /// *takes* from the transaction outputs in the index. Shorthand for calling
-    /// [`sent_and_received`] and subtracting sent from received.
-    ///
-    /// This calls [`SpkTxOutIndex::net_value`] internally.
-    ///
-    /// [`sent_and_received`]: Self::sent_and_received
-    pub fn net_value(&self, tx: &Transaction) -> i64 {
-        self.inner.net_value(tx)
-    }
-}
-
-impl<K: Clone + Ord + Debug> KeychainTxOutIndex<K> {
-    /// Return a reference to the internal map of keychain to descriptors.
-    pub fn keychains(&self) -> &BTreeMap<K, Descriptor<DescriptorPublicKey>> {
-        &self.keychains
-    }
-
-    /// Add a keychain to the tracker's `txout_index` with a descriptor to derive addresses.
-    ///
-    /// Adding a keychain means you will be able to derive new script pubkeys under that keychain
-    /// and the txout index will discover transaction outputs with those script pubkeys.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if a different `descriptor` is introduced to the same `keychain`.
-    pub fn add_keychain(&mut self, keychain: K, descriptor: Descriptor<DescriptorPublicKey>) {
-        let old_descriptor = &*self
-            .keychains
-            .entry(keychain.clone())
-            .or_insert_with(|| descriptor.clone());
-        assert_eq!(
-            &descriptor, old_descriptor,
-            "keychain already contains a different descriptor"
-        );
-        self.replenish_lookahead(&keychain, self.lookahead);
-    }
-
-    /// Get the lookahead setting.
-    ///
-    /// Refer to [`new`] for more information on the `lookahead`.
-    ///
-    /// [`new`]: Self::new
-    pub fn lookahead(&self) -> u32 {
-        self.lookahead
-    }
-
-    /// Store lookahead scripts until `target_index`.
-    ///
-    /// This does not change the `lookahead` setting.
-    pub fn lookahead_to_target(&mut self, keychain: &K, target_index: u32) {
-        let next_index = self.next_store_index(keychain);
-        if let Some(temp_lookahead) = target_index.checked_sub(next_index).filter(|&v| v > 0) {
-            self.replenish_lookahead(keychain, temp_lookahead);
-        }
-    }
-
-    fn replenish_lookahead(&mut self, keychain: &K, lookahead: u32) {
-        let descriptor = self.keychains.get(keychain).expect("keychain must exist");
-        let next_store_index = self.next_store_index(keychain);
-        let next_reveal_index = self.last_revealed.get(keychain).map_or(0, |v| *v + 1);
-
-        for (new_index, new_spk) in
-            SpkIterator::new_with_range(descriptor, next_store_index..next_reveal_index + lookahead)
-        {
-            let _inserted = self
-                .inner
-                .insert_spk((keychain.clone(), new_index), new_spk);
-            debug_assert!(_inserted, "replenish lookahead: must not have existing spk: keychain={:?}, lookahead={}, next_store_index={}, next_reveal_index={}", keychain, lookahead, next_store_index, next_reveal_index);
-        }
-    }
-
-    fn next_store_index(&self, keychain: &K) -> u32 {
-        self.inner()
-            .all_spks()
-            // This range is filtering out the spks with a keychain different than
-            // `keychain`. We don't use filter here as range is more optimized.
-            .range((keychain.clone(), u32::MIN)..(keychain.clone(), u32::MAX))
-            .last()
-            .map_or(0, |((_, index), _)| *index + 1)
-    }
-
-    /// Get an unbounded spk iterator over a given `keychain`.
-    ///
-    /// # Panics
-    ///
-    /// This will panic if the given `keychain`'s descriptor does not exist.
-    pub fn unbounded_spk_iter(&self, keychain: &K) -> SpkIterator<Descriptor<DescriptorPublicKey>> {
-        SpkIterator::new(
-            self.keychains
-                .get(keychain)
-                .expect("keychain does not exist")
-                .clone(),
-        )
-    }
-
-    /// Get unbounded spk iterators for all keychains.
-    pub fn all_unbounded_spk_iters(
-        &self,
-    ) -> BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>> {
-        self.keychains
-            .iter()
-            .map(|(k, descriptor)| (k.clone(), SpkIterator::new(descriptor.clone())))
-            .collect()
-    }
-
-    /// Iterate over revealed spks of all keychains.
-    pub fn revealed_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
-        self.keychains.keys().flat_map(|keychain| {
-            self.revealed_keychain_spks(keychain)
-                .map(|(i, spk)| (keychain.clone(), i, spk))
-        })
-    }
-
-    /// Iterate over revealed spks of the given `keychain`.
-    pub fn revealed_keychain_spks(
-        &self,
-        keychain: &K,
-    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
-        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
-        self.inner
-            .all_spks()
-            .range((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
-            .map(|((_, i), spk)| (*i, spk.as_script()))
-    }
-
-    /// Iterate over revealed, but unused, spks of all keychains.
-    pub fn unused_spks(&self) -> impl DoubleEndedIterator<Item = (K, u32, &Script)> + Clone {
-        self.keychains.keys().flat_map(|keychain| {
-            self.unused_keychain_spks(keychain)
-                .map(|(i, spk)| (keychain.clone(), i, spk))
-        })
-    }
-
-    /// Iterate over revealed, but unused, spks of the given `keychain`.
-    pub fn unused_keychain_spks(
-        &self,
-        keychain: &K,
-    ) -> impl DoubleEndedIterator<Item = (u32, &Script)> + Clone {
-        let next_i = self.last_revealed.get(keychain).map_or(0, |&i| i + 1);
-        self.inner
-            .unused_spks((keychain.clone(), u32::MIN)..(keychain.clone(), next_i))
-            .map(|((_, i), spk)| (*i, spk))
-    }
-
-    /// Get the next derivation index for `keychain`. The next index is the index after the last revealed
-    /// derivation index.
-    ///
-    /// The second field in the returned tuple represents whether the next derivation index is new.
-    /// There are two scenarios where the next derivation index is reused (not new):
-    ///
-    /// 1. The keychain's descriptor has no wildcard, and a script has already been revealed.
-    /// 2. The number of revealed scripts has already reached 2^31 (refer to BIP-32).
-    ///
-    /// Not checking the second field of the tuple may result in address reuse.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the `keychain` does not exist.
-    pub fn next_index(&self, keychain: &K) -> (u32, bool) {
-        let descriptor = self.keychains.get(keychain).expect("keychain must exist");
-        let last_index = self.last_revealed.get(keychain).cloned();
-
-        // we can only get the next index if the wildcard exists.
-        let has_wildcard = descriptor.has_wildcard();
-
-        match last_index {
-            // if there is no index, next_index is always 0.
-            None => (0, true),
-            // descriptors without wildcards can only have one index.
-            Some(_) if !has_wildcard => (0, false),
-            // derivation index must be < 2^31 (BIP-32).
-            Some(index) if index > BIP32_MAX_INDEX => {
-                unreachable!("index is out of bounds")
-            }
-            Some(index) if index == BIP32_MAX_INDEX => (index, false),
-            // get the next derivation index.
-            Some(index) => (index + 1, true),
-        }
-    }
-
-    /// Get the last derivation index that is revealed for each keychain.
-    ///
-    /// Keychains with no revealed indices will not be included in the returned [`BTreeMap`].
-    pub fn last_revealed_indices(&self) -> &BTreeMap<K, u32> {
-        &self.last_revealed
-    }
-
-    /// Get the last derivation index revealed for `keychain`.
-    pub fn last_revealed_index(&self, keychain: &K) -> Option<u32> {
-        self.last_revealed.get(keychain).cloned()
-    }
-
-    /// Convenience method to call [`Self::reveal_to_target`] on multiple keychains.
-    pub fn reveal_to_target_multi(
-        &mut self,
-        keychains: &BTreeMap<K, u32>,
-    ) -> (
-        BTreeMap<K, SpkIterator<Descriptor<DescriptorPublicKey>>>,
-        super::ChangeSet<K>,
-    ) {
-        let mut changeset = super::ChangeSet::default();
-        let mut spks = BTreeMap::new();
-
-        for (keychain, &index) in keychains {
-            let (new_spks, new_changeset) = self.reveal_to_target(keychain, index);
-            if !new_changeset.is_empty() {
-                spks.insert(keychain.clone(), new_spks);
-                changeset.append(new_changeset.clone());
-            }
-        }
-
-        (spks, changeset)
-    }
-
-    /// Reveals script pubkeys of the `keychain`'s descriptor **up to and including** the
-    /// `target_index`.
-    ///
-    /// If the `target_index` cannot be reached (due to the descriptor having no wildcard and/or
-    /// the `target_index` is in the hardened index range), this method will make a best-effort and
-    /// reveal up to the last possible index.
-    ///
-    /// This returns an iterator of newly revealed indices (alongside their scripts) and a
-    /// [`super::ChangeSet`], which reports updates to the latest revealed index. If no new script
-    /// pubkeys are revealed, then both of these will be empty.
-    ///
-    /// # Panics
-    ///
-    /// Panics if `keychain` does not exist.
-    pub fn reveal_to_target(
-        &mut self,
-        keychain: &K,
-        target_index: u32,
-    ) -> (
-        SpkIterator<Descriptor<DescriptorPublicKey>>,
-        super::ChangeSet<K>,
-    ) {
-        let descriptor = self.keychains.get(keychain).expect("keychain must exist");
-        let has_wildcard = descriptor.has_wildcard();
-
-        let target_index = if has_wildcard { target_index } else { 0 };
-        let next_reveal_index = self
-            .last_revealed
-            .get(keychain)
-            .map_or(0, |index| *index + 1);
-
-        debug_assert!(next_reveal_index + self.lookahead >= self.next_store_index(keychain));
-
-        // If the target_index is already revealed, we are done
-        if next_reveal_index > target_index {
-            return (
-                SpkIterator::new_with_range(
-                    descriptor.clone(),
-                    next_reveal_index..next_reveal_index,
-                ),
-                super::ChangeSet::default(),
-            );
-        }
-
-        // We range over the indexes that are not stored and insert their spks in the index.
-        // Indexes from next_reveal_index to next_reveal_index + lookahead are already stored (due
-        // to lookahead), so we only range from next_reveal_index + lookahead to target + lookahead
-        let range = next_reveal_index + self.lookahead..=target_index + self.lookahead;
-        for (new_index, new_spk) in SpkIterator::new_with_range(descriptor, range) {
-            let _inserted = self
-                .inner
-                .insert_spk((keychain.clone(), new_index), new_spk);
-            debug_assert!(_inserted, "must not have existing spk");
-            debug_assert!(
-                has_wildcard || new_index == 0,
-                "non-wildcard descriptors must not iterate past index 0"
-            );
-        }
-
-        let _old_index = self.last_revealed.insert(keychain.clone(), target_index);
-        debug_assert!(_old_index < Some(target_index));
-        (
-            SpkIterator::new_with_range(descriptor.clone(), next_reveal_index..target_index + 1),
-            super::ChangeSet(core::iter::once((keychain.clone(), target_index)).collect()),
-        )
-    }
-
-    /// Attempts to reveal the next script pubkey for `keychain`.
-    ///
-    /// Returns the derivation index of the revealed script pubkey, the revealed script pubkey and a
-    /// [`super::ChangeSet`] which represents changes in the last revealed index (if any).
-    ///
-    /// When a new script cannot be revealed, we return the last revealed script and an empty
-    /// [`super::ChangeSet`]. There are two scenarios when a new script pubkey cannot be derived:
-    ///
-    ///  1. The descriptor has no wildcard and already has one script revealed.
-    ///  2. The descriptor has already revealed scripts up to the numeric bound.
-    ///
-    /// # Panics
-    ///
-    /// Panics if the `keychain` does not exist.
-    pub fn reveal_next_spk(&mut self, keychain: &K) -> ((u32, &Script), super::ChangeSet<K>) {
-        let (next_index, _) = self.next_index(keychain);
-        let changeset = self.reveal_to_target(keychain, next_index).1;
-        let script = self
-            .inner
-            .spk_at_index(&(keychain.clone(), next_index))
-            .expect("script must already be stored");
-        ((next_index, script), changeset)
-    }
-
-    /// Gets the next unused script pubkey in the keychain. I.e., the script pubkey with the lowest
-    /// index that has not been used yet.
-    ///
-    /// This will derive and reveal a new script pubkey if no more unused script pubkeys exist.
-    ///
-    /// If the descriptor has no wildcard and already has a used script pubkey or if a descriptor
-    /// has used all scripts up to the derivation bounds, then the last derived script pubkey will be
-    /// returned.
-    ///
-    /// # Panics
-    ///
-    /// Panics if `keychain` has never been added to the index
-    pub fn next_unused_spk(&mut self, keychain: &K) -> ((u32, &Script), super::ChangeSet<K>) {
-        let need_new = self.unused_keychain_spks(keychain).next().is_none();
-        // this rather strange branch is needed because of some lifetime issues
-        if need_new {
-            self.reveal_next_spk(keychain)
-        } else {
-            (
-                self.unused_keychain_spks(keychain)
-                    .next()
-                    .expect("we already know next exists"),
-                super::ChangeSet::default(),
-            )
-        }
-    }
-
-    /// Iterate over all [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
-    /// `keychain`.
-    ///
-    /// Use [`keychain_outpoints_in_range`](KeychainTxOutIndex::keychain_outpoints_in_range) to
-    /// iterate over a specific derivation range.
-    pub fn keychain_outpoints(
-        &self,
-        keychain: &K,
-    ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
-        self.keychain_outpoints_in_range(keychain, ..)
-    }
-
-    /// Iterate over [`OutPoint`]s that point to `TxOut`s with script pubkeys derived from
-    /// `keychain` in a given derivation `range`.
-    pub fn keychain_outpoints_in_range(
-        &self,
-        keychain: &K,
-        range: impl RangeBounds<u32>,
-    ) -> impl DoubleEndedIterator<Item = (u32, OutPoint)> + '_ {
-        let start = match range.start_bound() {
-            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
-            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
-            Bound::Unbounded => Bound::Unbounded,
-        };
-        let end = match range.end_bound() {
-            Bound::Included(i) => Bound::Included((keychain.clone(), *i)),
-            Bound::Excluded(i) => Bound::Excluded((keychain.clone(), *i)),
-            Bound::Unbounded => Bound::Unbounded,
-        };
-        self.inner
-            .outputs_in_range((start, end))
-            .map(|((_, i), op)| (*i, op))
-    }
-
-    /// Returns the highest derivation index of the `keychain` where [`KeychainTxOutIndex`] has
-    /// found a [`TxOut`] with it's script pubkey.
-    pub fn last_used_index(&self, keychain: &K) -> Option<u32> {
-        self.keychain_outpoints(keychain).last().map(|(i, _)| i)
-    }
-
-    /// Returns the highest derivation index of each keychain that [`KeychainTxOutIndex`] has found
-    /// a [`TxOut`] with it's script pubkey.
-    pub fn last_used_indices(&self) -> BTreeMap<K, u32> {
-        self.keychains
-            .iter()
-            .filter_map(|(keychain, _)| {
-                self.last_used_index(keychain)
-                    .map(|index| (keychain.clone(), index))
-            })
-            .collect()
-    }
-
-    /// Applies the derivation changeset to the [`KeychainTxOutIndex`], extending the number of
-    /// derived scripts per keychain, as specified in the `changeset`.
-    pub fn apply_changeset(&mut self, changeset: super::ChangeSet<K>) {
-        let _ = self.reveal_to_target_multi(&changeset.0);
-    }
-}

crates/chain/src/lib.rs

@@ -21,13 +21,15 @@
 #![warn(missing_docs)]
 
 pub use bitcoin;
-mod spk_txout_index;
-pub use spk_txout_index::*;
+mod balance;
+pub use balance::*;
 mod chain_data;
 pub use chain_data::*;
 pub mod indexed_tx_graph;
 pub use indexed_tx_graph::IndexedTxGraph;
-pub mod keychain;
+pub mod indexer;
+pub use indexer::spk_txout;
+pub use indexer::Indexer;
 pub mod local_chain;
 mod tx_data_traits;
 pub mod tx_graph;
@@ -46,22 +48,25 @@ pub use miniscript;
 #[cfg(feature = "miniscript")]
 mod descriptor_ext;
 #[cfg(feature = "miniscript")]
-pub use descriptor_ext::DescriptorExt;
+pub use descriptor_ext::{DescriptorExt, DescriptorId};
 #[cfg(feature = "miniscript")]
 mod spk_iter;
 #[cfg(feature = "miniscript")]
+pub use indexer::keychain_txout;
+#[cfg(feature = "miniscript")]
 pub use spk_iter::*;
+#[cfg(feature = "rusqlite")]
+pub mod rusqlite_impl;
+pub mod spk_client;
 
 #[allow(unused_imports)]
 #[macro_use]
 extern crate alloc;
-
+#[cfg(feature = "rusqlite")]
+pub extern crate rusqlite_crate as rusqlite;
 #[cfg(feature = "serde")]
 pub extern crate serde_crate as serde;
 
-#[cfg(feature = "bincode")]
-extern crate bincode;
-
 #[cfg(feature = "std")]
 #[macro_use]
 extern crate std;
@@ -99,3 +104,25 @@ pub mod collections {
 
 /// How many confirmations are needed f or a coinbase output to be spent.
 pub const COINBASE_MATURITY: u32 = 100;
+
+/// A tuple of keychain index and `T` representing the indexed value.
+pub type Indexed<T> = (u32, T);
+/// A tuple of keychain `K`, derivation index (`u32`) and a `T` associated with them.
+pub type KeychainIndexed<K, T> = ((K, u32), T);
+
+/// A wrapper that we use to impl remote traits for types in our crate or dependency crates.
+pub struct Impl<T>(pub T);
+
+impl<T> From<T> for Impl<T> {
+    fn from(value: T) -> Self {
+        Self(value)
+    }
+}
+
+impl<T> core::ops::Deref for Impl<T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}

crates/chain/src/local_chain.rs

@@ -1,19 +1,14 @@
 //! The [`LocalChain`] is a local implementation of [`ChainOracle`].
 
 use core::convert::Infallible;
+use core::ops::RangeBounds;
 
 use crate::collections::BTreeMap;
-use crate::{BlockId, ChainOracle};
+use crate::{BlockId, ChainOracle, Merge};
 use alloc::sync::Arc;
 use bitcoin::block::Header;
 use bitcoin::BlockHash;
 
-/// The [`ChangeSet`] represents changes to [`LocalChain`].
-///
-/// The key represents the block height, and the value either represents added a new [`CheckPoint`]
-/// (if [`Some`]), or removing a [`CheckPoint`] (if [`None`]).
-pub type ChangeSet = BTreeMap<u32, Option<BlockHash>>;
-
 /// A [`LocalChain`] checkpoint is used to find the agreement point between two chains and as a
 /// transaction anchor.
 ///
@@ -34,6 +29,14 @@ struct CPInner {
     prev: Option<Arc<CPInner>>,
 }
 
+impl PartialEq for CheckPoint {
+    fn eq(&self, other: &Self) -> bool {
+        let self_cps = self.iter().map(|cp| cp.block_id());
+        let other_cps = other.iter().map(|cp| cp.block_id());
+        self_cps.eq(other_cps)
+    }
+}
+
 impl CheckPoint {
     /// Construct a new base block at the front of a linked list.
     pub fn new(block: BlockId) -> Self {
@@ -87,16 +90,6 @@ impl CheckPoint {
             .expect("must construct checkpoint")
     }
 
-    /// Convenience method to convert the [`CheckPoint`] into an [`Update`].
-    ///
-    /// For more information, refer to [`Update`].
-    pub fn into_update(self, introduce_older_blocks: bool) -> Update {
-        Update {
-            tip: self,
-            introduce_older_blocks,
-        }
-    }
-
     /// Puts another checkpoint onto the linked list representing the blockchain.
     ///
     /// Returns an `Err(self)` if the block you are pushing on is not at a greater height that the one you
@@ -148,6 +141,112 @@ impl CheckPoint {
     pub fn iter(&self) -> CheckPointIter {
         self.clone().into_iter()
     }
+
+    /// Get checkpoint at `height`.
+    ///
+    /// Returns `None` if checkpoint at `height` does not exist`.
+    pub fn get(&self, height: u32) -> Option<Self> {
+        self.range(height..=height).next()
+    }
+
+    /// Iterate checkpoints over a height range.
+    ///
+    /// Note that we always iterate checkpoints in reverse height order (iteration starts at tip
+    /// height).
+    pub fn range<R>(&self, range: R) -> impl Iterator<Item = CheckPoint>
+    where
+        R: RangeBounds<u32>,
+    {
+        let start_bound = range.start_bound().cloned();
+        let end_bound = range.end_bound().cloned();
+        self.iter()
+            .skip_while(move |cp| match end_bound {
+                core::ops::Bound::Included(inc_bound) => cp.height() > inc_bound,
+                core::ops::Bound::Excluded(exc_bound) => cp.height() >= exc_bound,
+                core::ops::Bound::Unbounded => false,
+            })
+            .take_while(move |cp| match start_bound {
+                core::ops::Bound::Included(inc_bound) => cp.height() >= inc_bound,
+                core::ops::Bound::Excluded(exc_bound) => cp.height() > exc_bound,
+                core::ops::Bound::Unbounded => true,
+            })
+    }
+
+    /// Inserts `block_id` at its height within the chain.
+    ///
+    /// The effect of `insert` depends on whether a height already exists. If it doesn't the
+    /// `block_id` we inserted and all pre-existing blocks higher than it will be re-inserted after
+    /// it. If the height already existed and has a conflicting block hash then it will be purged
+    /// along with all block followin it. The returned chain will have a tip of the `block_id`
+    /// passed in. Of course, if the `block_id` was already present then this just returns `self`.
+    #[must_use]
+    pub fn insert(self, block_id: BlockId) -> Self {
+        assert_ne!(block_id.height, 0, "cannot insert the genesis block");
+
+        let mut cp = self.clone();
+        let mut tail = vec![];
+        let base = loop {
+            if cp.height() == block_id.height {
+                if cp.hash() == block_id.hash {
+                    return self;
+                }
+                // if we have a conflict we just return the inserted block because the tail is by
+                // implication invalid.
+                tail = vec![];
+                break cp.prev().expect("can't be called on genesis block");
+            }
+
+            if cp.height() < block_id.height {
+                break cp;
+            }
+
+            tail.push(cp.block_id());
+            cp = cp.prev().expect("will break before genesis block");
+        };
+
+        base.extend(core::iter::once(block_id).chain(tail.into_iter().rev()))
+            .expect("tail is in order")
+    }
+
+    /// Apply `changeset` to the checkpoint.
+    fn apply_changeset(mut self, changeset: &ChangeSet) -> Result<CheckPoint, MissingGenesisError> {
+        if let Some(start_height) = changeset.blocks.keys().next().cloned() {
+            // changes after point of agreement
+            let mut extension = BTreeMap::default();
+            // point of agreement
+            let mut base: Option<CheckPoint> = None;
+
+            for cp in self.iter() {
+                if cp.height() >= start_height {
+                    extension.insert(cp.height(), cp.hash());
+                } else {
+                    base = Some(cp);
+                    break;
+                }
+            }
+
+            for (&height, &hash) in &changeset.blocks {
+                match hash {
+                    Some(hash) => {
+                        extension.insert(height, hash);
+                    }
+                    None => {
+                        extension.remove(&height);
+                    }
+                };
+            }
+
+            let new_tip = match base {
+                Some(base) => base
+                    .extend(extension.into_iter().map(BlockId::from))
+                    .expect("extension is strictly greater than base"),
+                None => LocalChain::from_blocks(extension)?.tip(),
+            };
+            self = new_tip;
+        }
+
+        Ok(self)
+    }
 }
 
 /// Iterates over checkpoints backwards.
@@ -160,7 +259,7 @@ impl Iterator for CheckPointIter {
 
     fn next(&mut self) -> Option<Self::Item> {
         let current = self.current.clone()?;
-        self.current = current.prev.clone();
+        self.current.clone_from(&current.prev);
         Some(CheckPoint(current))
     }
 }
@@ -176,48 +275,10 @@ impl IntoIterator for CheckPoint {
     }
 }
 
-/// Used to update [`LocalChain`].
-///
-/// This is used as input for [`LocalChain::apply_update`]. It contains the update's chain `tip` and
-/// a flag `introduce_older_blocks` which signals whether this update intends to introduce missing
-/// blocks to the original chain.
-///
-/// Block-by-block syncing mechanisms would typically create updates that builds upon the previous
-/// tip. In this case, `introduce_older_blocks` would be `false`.
-///
-/// Script-pubkey based syncing mechanisms may not introduce transactions in a chronological order
-/// so some updates require introducing older blocks (to anchor older transactions). For
-/// script-pubkey based syncing, `introduce_older_blocks` would typically be `true`.
-#[derive(Debug, Clone)]
-pub struct Update {
-    /// The update chain's new tip.
-    pub tip: CheckPoint,
-
-    /// Whether the update allows for introducing older blocks.
-    ///
-    /// Refer to [struct-level documentation] for more.
-    ///
-    /// [struct-level documentation]: Update
-    pub introduce_older_blocks: bool,
-}
-
 /// This is a local implementation of [`ChainOracle`].
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, PartialEq)]
 pub struct LocalChain {
     tip: CheckPoint,
-    index: BTreeMap<u32, BlockHash>,
-}
-
-impl PartialEq for LocalChain {
-    fn eq(&self, other: &Self) -> bool {
-        self.index == other.index
-    }
-}
-
-impl From<LocalChain> for BTreeMap<u32, BlockHash> {
-    fn from(value: LocalChain) -> Self {
-        value.index
-    }
 }
 
 impl ChainOracle for LocalChain {
@@ -228,18 +289,16 @@ impl ChainOracle for LocalChain {
         block: BlockId,
         chain_tip: BlockId,
     ) -> Result<Option<bool>, Self::Error> {
-        if block.height > chain_tip.height {
-            return Ok(None);
+        let chain_tip_cp = match self.tip.get(chain_tip.height) {
+            // we can only determine whether `block` is in chain of `chain_tip` if `chain_tip` can
+            // be identified in chain
+            Some(cp) if cp.hash() == chain_tip.hash => cp,
+            _ => return Ok(None),
+        };
+        match chain_tip_cp.get(block.height) {
+            Some(cp) => Ok(Some(cp.hash() == block.hash)),
+            None => Ok(None),
         }
-        Ok(
-            match (
-                self.index.get(&block.height),
-                self.index.get(&chain_tip.height),
-            ) {
-                (Some(cp), Some(tip_cp)) => Some(*cp == block.hash && *tip_cp == chain_tip.hash),
-                _ => None,
-            },
-        )
     }
 
     fn get_chain_tip(&self) -> Result<BlockId, Self::Error> {
@@ -250,7 +309,7 @@ impl ChainOracle for LocalChain {
 impl LocalChain {
     /// Get the genesis hash.
     pub fn genesis_hash(&self) -> BlockHash {
-        self.index.get(&0).copied().expect("must have genesis hash")
+        self.tip.get(0).expect("genesis must exist").hash()
     }
 
     /// Construct [`LocalChain`] from genesis `hash`.
@@ -259,7 +318,6 @@ impl LocalChain {
         let height = 0;
         let chain = Self {
             tip: CheckPoint::new(BlockId { height, hash }),
-            index: core::iter::once((height, hash)).collect(),
         };
         let changeset = chain.initial_changeset();
         (chain, changeset)
@@ -267,7 +325,7 @@ impl LocalChain {
 
     /// Construct a [`LocalChain`] from an initial `changeset`.
     pub fn from_changeset(changeset: ChangeSet) -> Result<Self, MissingGenesisError> {
-        let genesis_entry = changeset.get(&0).copied().flatten();
+        let genesis_entry = changeset.blocks.get(&0).copied().flatten();
         let genesis_hash = match genesis_entry {
             Some(hash) => hash,
             None => return Err(MissingGenesisError),
@@ -276,7 +334,6 @@ impl LocalChain {
         let (mut chain, _) = Self::from_genesis_hash(genesis_hash);
         chain.apply_changeset(&changeset)?;
 
-        debug_assert!(chain._check_index_is_consistent_with_tip());
         debug_assert!(chain._check_changeset_is_applied(&changeset));
 
         Ok(chain)
@@ -284,18 +341,11 @@ impl LocalChain {
 
     /// Construct a [`LocalChain`] from a given `checkpoint` tip.
     pub fn from_tip(tip: CheckPoint) -> Result<Self, MissingGenesisError> {
-        let mut chain = Self {
-            tip,
-            index: BTreeMap::new(),
-        };
-        chain.reindex(0);
-
-        if chain.index.get(&0).copied().is_none() {
+        let genesis_cp = tip.iter().last().expect("must have at least one element");
+        if genesis_cp.height() != 0 {
             return Err(MissingGenesisError);
         }
-
-        debug_assert!(chain._check_index_is_consistent_with_tip());
-        Ok(chain)
+        Ok(Self { tip })
     }
 
     /// Constructs a [`LocalChain`] from a [`BTreeMap`] of height to [`BlockHash`].
@@ -308,7 +358,6 @@ impl LocalChain {
         }
 
         let mut tip: Option<CheckPoint> = None;
-
         for block in &blocks {
             match tip {
                 Some(curr) => {
@@ -321,13 +370,9 @@ impl LocalChain {
             }
         }
 
-        let chain = Self {
-            index: blocks,
+        Ok(Self {
             tip: tip.expect("already checked to have genesis"),
-        };
-
-        debug_assert!(chain._check_index_is_consistent_with_tip());
-        Ok(chain)
+        })
     }
 
     /// Get the highest checkpoint.
@@ -337,36 +382,22 @@ impl LocalChain {
 
     /// Applies the given `update` to the chain.
     ///
-    /// The method returns [`ChangeSet`] on success. This represents the applied changes to `self`.
+    /// The method returns [`ChangeSet`] on success. This represents the changes applied to `self`.
     ///
     /// There must be no ambiguity about which of the existing chain's blocks are still valid and
     /// which are now invalid. That is, the new chain must implicitly connect to a definite block in
     /// the existing chain and invalidate the block after it (if it exists) by including a block at
     /// the same height but with a different hash to explicitly exclude it as a connection point.
     ///
-    /// Additionally, an empty chain can be updated with any chain, and a chain with a single block
-    /// can have it's block invalidated by an update chain with a block at the same height but
-    /// different hash.
-    ///
     /// # Errors
     ///
     /// An error will occur if the update does not correctly connect with `self`.
     ///
-    /// Refer to [`Update`] for more about the update struct.
-    ///
     /// [module-level documentation]: crate::local_chain
-    pub fn apply_update(&mut self, update: Update) -> Result<ChangeSet, CannotConnectError> {
-        let changeset = merge_chains(
-            self.tip.clone(),
-            update.tip.clone(),
-            update.introduce_older_blocks,
-        )?;
-        // `._check_index_is_consistent_with_tip` and `._check_changeset_is_applied` is called in
-        // `.apply_changeset`
-        self.apply_changeset(&changeset)
-            .map_err(|_| CannotConnectError {
-                try_include_height: 0,
-            })?;
+    pub fn apply_update(&mut self, update: CheckPoint) -> Result<ChangeSet, CannotConnectError> {
+        let (new_tip, changeset) = merge_chains(self.tip.clone(), update)?;
+        self.tip = new_tip;
+        self._check_changeset_is_applied(&changeset);
         Ok(changeset)
     }
 
@@ -418,11 +449,8 @@ impl LocalChain {
             conn => Some(conn),
         };
 
-        let update = Update {
-            tip: CheckPoint::from_block_ids([conn, prev, Some(this)].into_iter().flatten())
-                .expect("block ids must be in order"),
-            introduce_older_blocks: false,
-        };
+        let update = CheckPoint::from_block_ids([conn, prev, Some(this)].into_iter().flatten())
+            .expect("block ids must be in order");
 
         self.apply_update(update)
             .map_err(ApplyHeaderError::CannotConnect)
@@ -461,45 +489,10 @@ impl LocalChain {
 
     /// Apply the given `changeset`.
     pub fn apply_changeset(&mut self, changeset: &ChangeSet) -> Result<(), MissingGenesisError> {
-        if let Some(start_height) = changeset.keys().next().cloned() {
-            // changes after point of agreement
-            let mut extension = BTreeMap::default();
-            // point of agreement
-            let mut base: Option<CheckPoint> = None;
-
-            for cp in self.iter_checkpoints() {
-                if cp.height() >= start_height {
-                    extension.insert(cp.height(), cp.hash());
-                } else {
-                    base = Some(cp);
-                    break;
-                }
-            }
-
-            for (&height, &hash) in changeset {
-                match hash {
-                    Some(hash) => {
-                        extension.insert(height, hash);
-                    }
-                    None => {
-                        extension.remove(&height);
-                    }
-                };
-            }
-
-            let new_tip = match base {
-                Some(base) => base
-                    .extend(extension.into_iter().map(BlockId::from))
-                    .expect("extension is strictly greater than base"),
-                None => LocalChain::from_blocks(extension)?.tip(),
-            };
-            self.tip = new_tip;
-            self.reindex(start_height);
-
-            debug_assert!(self._check_index_is_consistent_with_tip());
-            debug_assert!(self._check_changeset_is_applied(changeset));
-        }
-
+        let old_tip = self.tip.clone();
+        let new_tip = old_tip.apply_changeset(changeset)?;
+        self.tip = new_tip;
+        debug_assert!(self._check_changeset_is_applied(changeset));
         Ok(())
     }
 
@@ -509,25 +502,27 @@ impl LocalChain {
     ///
     /// Replacing the block hash of an existing checkpoint will result in an error.
     pub fn insert_block(&mut self, block_id: BlockId) -> Result<ChangeSet, AlterCheckPointError> {
-        if let Some(&original_hash) = self.index.get(&block_id.height) {
+        if let Some(original_cp) = self.tip.get(block_id.height) {
+            let original_hash = original_cp.hash();
             if original_hash != block_id.hash {
                 return Err(AlterCheckPointError {
                     height: block_id.height,
                     original_hash,
                     update_hash: Some(block_id.hash),
                 });
-            } else {
-                return Ok(ChangeSet::default());
             }
+            return Ok(ChangeSet::default());
         }
 
         let mut changeset = ChangeSet::default();
-        changeset.insert(block_id.height, Some(block_id.hash));
+        changeset
+            .blocks
+            .insert(block_id.height, Some(block_id.hash));
         self.apply_changeset(&changeset)
             .map_err(|_| AlterCheckPointError {
                 height: 0,
                 original_hash: self.genesis_hash(),
-                update_hash: changeset.get(&0).cloned().flatten(),
+                update_hash: changeset.blocks.get(&0).cloned().flatten(),
             })?;
         Ok(changeset)
     }
@@ -542,33 +537,44 @@ impl LocalChain {
     /// This will fail with [`MissingGenesisError`] if the caller attempts to disconnect from the
     /// genesis block.
     pub fn disconnect_from(&mut self, block_id: BlockId) -> Result<ChangeSet, MissingGenesisError> {
-        if self.index.get(&block_id.height) != Some(&block_id.hash) {
-            return Ok(ChangeSet::default());
-        }
-
-        let changeset = self
-            .index
-            .range(block_id.height..)
-            .map(|(&height, _)| (height, None))
-            .collect::<ChangeSet>();
-        self.apply_changeset(&changeset).map(|_| changeset)
-    }
-
-    /// Reindex the heights in the chain from (and including) `from` height
-    fn reindex(&mut self, from: u32) {
-        let _ = self.index.split_off(&from);
-        for cp in self.iter_checkpoints() {
-            if cp.height() < from {
+        let mut remove_from = Option::<CheckPoint>::None;
+        let mut changeset = ChangeSet::default();
+        for cp in self.tip().iter() {
+            let cp_id = cp.block_id();
+            if cp_id.height < block_id.height {
                 break;
             }
-            self.index.insert(cp.height(), cp.hash());
+            changeset.blocks.insert(cp_id.height, None);
+            if cp_id == block_id {
+                remove_from = Some(cp);
+            }
         }
+        self.tip = match remove_from.map(|cp| cp.prev()) {
+            // The checkpoint below the earliest checkpoint to remove will be the new tip.
+            Some(Some(new_tip)) => new_tip,
+            // If there is no checkpoint below the earliest checkpoint to remove, it means the
+            // "earliest checkpoint to remove" is the genesis block. We disallow removing the
+            // genesis block.
+            Some(None) => return Err(MissingGenesisError),
+            // If there is nothing to remove, we return an empty changeset.
+            None => return Ok(ChangeSet::default()),
+        };
+        Ok(changeset)
     }
 
     /// Derives an initial [`ChangeSet`], meaning that it can be applied to an empty chain to
     /// recover the current chain.
     pub fn initial_changeset(&self) -> ChangeSet {
-        self.index.iter().map(|(k, v)| (*k, Some(*v))).collect()
+        ChangeSet {
+            blocks: self
+                .tip
+                .iter()
+                .map(|cp| {
+                    let block_id = cp.block_id();
+                    (block_id.height, Some(block_id.hash))
+                })
+                .collect(),
+        }
     }
 
     /// Iterate over checkpoints in descending height order.
@@ -578,28 +584,101 @@ impl LocalChain {
         }
     }
 
-    /// Get a reference to the internal index mapping the height to block hash.
-    pub fn blocks(&self) -> &BTreeMap<u32, BlockHash> {
-        &self.index
-    }
-
-    fn _check_index_is_consistent_with_tip(&self) -> bool {
-        let tip_history = self
-            .tip
-            .iter()
-            .map(|cp| (cp.height(), cp.hash()))
-            .collect::<BTreeMap<_, _>>();
-        self.index == tip_history
-    }
-
     fn _check_changeset_is_applied(&self, changeset: &ChangeSet) -> bool {
-        for (height, exp_hash) in changeset {
-            if self.index.get(height) != exp_hash.as_ref() {
-                return false;
+        let mut curr_cp = self.tip.clone();
+        for (height, exp_hash) in changeset.blocks.iter().rev() {
+            match curr_cp.get(*height) {
+                Some(query_cp) => {
+                    if query_cp.height() != *height || Some(query_cp.hash()) != *exp_hash {
+                        return false;
+                    }
+                    curr_cp = query_cp;
+                }
+                None => {
+                    if exp_hash.is_some() {
+                        return false;
+                    }
+                }
             }
         }
         true
     }
+
+    /// Get checkpoint at given `height` (if it exists).
+    ///
+    /// This is a shorthand for calling [`CheckPoint::get`] on the [`tip`].
+    ///
+    /// [`tip`]: LocalChain::tip
+    pub fn get(&self, height: u32) -> Option<CheckPoint> {
+        self.tip.get(height)
+    }
+
+    /// Iterate checkpoints over a height range.
+    ///
+    /// Note that we always iterate checkpoints in reverse height order (iteration starts at tip
+    /// height).
+    ///
+    /// This is a shorthand for calling [`CheckPoint::range`] on the [`tip`].
+    ///
+    /// [`tip`]: LocalChain::tip
+    pub fn range<R>(&self, range: R) -> impl Iterator<Item = CheckPoint>
+    where
+        R: RangeBounds<u32>,
+    {
+        self.tip.range(range)
+    }
+}
+
+/// The [`ChangeSet`] represents changes to [`LocalChain`].
+#[derive(Debug, Default, Clone, PartialEq)]
+#[cfg_attr(
+    feature = "serde",
+    derive(serde::Deserialize, serde::Serialize),
+    serde(crate = "serde_crate")
+)]
+pub struct ChangeSet {
+    /// Changes to the [`LocalChain`] blocks.
+    ///
+    /// The key represents the block height, and the value either represents added a new [`CheckPoint`]
+    /// (if [`Some`]), or removing a [`CheckPoint`] (if [`None`]).
+    pub blocks: BTreeMap<u32, Option<BlockHash>>,
+}
+
+impl Merge for ChangeSet {
+    fn merge(&mut self, other: Self) {
+        Merge::merge(&mut self.blocks, other.blocks)
+    }
+
+    fn is_empty(&self) -> bool {
+        self.blocks.is_empty()
+    }
+}
+
+impl<B: IntoIterator<Item = (u32, Option<BlockHash>)>> From<B> for ChangeSet {
+    fn from(blocks: B) -> Self {
+        Self {
+            blocks: blocks.into_iter().collect(),
+        }
+    }
+}
+
+impl FromIterator<(u32, Option<BlockHash>)> for ChangeSet {
+    fn from_iter<T: IntoIterator<Item = (u32, Option<BlockHash>)>>(iter: T) -> Self {
+        Self {
+            blocks: iter.into_iter().collect(),
+        }
+    }
+}
+
+impl FromIterator<(u32, BlockHash)> for ChangeSet {
+    fn from_iter<T: IntoIterator<Item = (u32, BlockHash)>>(iter: T) -> Self {
+        Self {
+            blocks: iter
+                .into_iter()
+                .map(|(height, hash)| (height, Some(hash)))
+                .collect(),
+        }
+    }
 }
 
 /// An error which occurs when a [`LocalChain`] is constructed without a genesis checkpoint.
@@ -693,14 +772,17 @@ impl core::fmt::Display for ApplyHeaderError {
 #[cfg(feature = "std")]
 impl std::error::Error for ApplyHeaderError {}
 
+/// Applies `update_tip` onto `original_tip`.
+///
+/// On success, a tuple is returned `(changeset, can_replace)`. If `can_replace` is true, then the
+/// `update_tip` can replace the `original_tip`.
 fn merge_chains(
     original_tip: CheckPoint,
     update_tip: CheckPoint,
-    introduce_older_blocks: bool,
-) -> Result<ChangeSet, CannotConnectError> {
+) -> Result<(CheckPoint, ChangeSet), CannotConnectError> {
     let mut changeset = ChangeSet::default();
-    let mut orig = original_tip.into_iter();
-    let mut update = update_tip.into_iter();
+    let mut orig = original_tip.iter();
+    let mut update = update_tip.iter();
     let mut curr_orig = None;
     let mut curr_update = None;
     let mut prev_orig: Option<CheckPoint> = None;
@@ -709,6 +791,12 @@ fn merge_chains(
     let mut prev_orig_was_invalidated = false;
     let mut potentially_invalidated_heights = vec![];
 
+    // If we can, we want to return the update tip as the new tip because this allows checkpoints
+    // in multiple locations to keep the same `Arc` pointers when they are being updated from each
+    // other using this function. We can do this as long as long as the update contains every
+    // block's height of the original chain.
+    let mut is_update_height_superset_of_original = true;
+
     // To find the difference between the new chain and the original we iterate over both of them
     // from the tip backwards in tandem. We always dealing with the highest one from either chain
     // first and move to the next highest. The crucial logic is applied when they have blocks at the
@@ -724,7 +812,7 @@ fn merge_chains(
         match (curr_orig.as_ref(), curr_update.as_ref()) {
             // Update block that doesn't exist in the original chain
             (o, Some(u)) if Some(u.height()) > o.map(|o| o.height()) => {
-                changeset.insert(u.height(), Some(u.hash()));
+                changeset.blocks.insert(u.height(), Some(u.hash()));
                 prev_update = curr_update.take();
             }
             // Original block that isn't in the update
@@ -734,6 +822,8 @@ fn merge_chains(
                 prev_orig_was_invalidated = false;
                 prev_orig = curr_orig.take();
 
+                is_update_height_superset_of_original = false;
+
                 // OPTIMIZATION: we have run out of update blocks so we don't need to continue
                 // iterating because there's no possibility of adding anything to changeset.
                 if u.is_none() {
@@ -756,19 +846,27 @@ fn merge_chains(
                     }
                     point_of_agreement_found = true;
                     prev_orig_was_invalidated = false;
-                    // OPTIMIZATION 1 -- If we know that older blocks cannot be introduced without
-                    // invalidation, we can break after finding the point of agreement.
                     // OPTIMIZATION 2 -- if we have the same underlying pointer at this point, we
                     // can guarantee that no older blocks are introduced.
-                    if !introduce_older_blocks || Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) {
-                        return Ok(changeset);
+                    if Arc::as_ptr(&o.0) == Arc::as_ptr(&u.0) {
+                        if is_update_height_superset_of_original {
+                            return Ok((update_tip, changeset));
+                        } else {
+                            let new_tip =
+                                original_tip.apply_changeset(&changeset).map_err(|_| {
+                                    CannotConnectError {
+                                        try_include_height: 0,
+                                    }
+                                })?;
+                            return Ok((new_tip, changeset));
+                        }
                     }
                 } else {
                     // We have an invalidation height so we set the height to the updated hash and
                     // also purge all the original chain block hashes above this block.
-                    changeset.insert(u.height(), Some(u.hash()));
+                    changeset.blocks.insert(u.height(), Some(u.hash()));
                     for invalidated_height in potentially_invalidated_heights.drain(..) {
-                        changeset.insert(invalidated_height, None);
+                        changeset.blocks.insert(invalidated_height, None);
                     }
                     prev_orig_was_invalidated = true;
                 }
@@ -795,5 +893,10 @@ fn merge_chains(
         }
     }
 
-    Ok(changeset)
+    let new_tip = original_tip
+        .apply_changeset(&changeset)
+        .map_err(|_| CannotConnectError {
+            try_include_height: 0,
+        })?;
+    Ok((new_tip, changeset))
 }

crates/chain/src/persist.rs

@@ -1,109 +1,169 @@
-use core::convert::Infallible;
+use core::{
+    future::Future,
+    ops::{Deref, DerefMut},
+    pin::Pin,
+};
 
-use crate::Append;
+use alloc::boxed::Box;
 
-/// `Persist` wraps a [`PersistBackend`] (`B`) to create a convenient staging area for changes (`C`)
-/// before they are persisted.
+use crate::Merge;
+
+/// Represents a type that contains staged changes.
+pub trait Staged {
+    /// Type for staged changes.
+    type ChangeSet: Merge;
+
+    /// Get mutable reference of staged changes.
+    fn staged(&mut self) -> &mut Self::ChangeSet;
+}
+
+/// Trait that persists the type with `Db`.
 ///
-/// Not all changes to the in-memory representation needs to be written to disk right away, so
-/// [`Persist::stage`] can be used to *stage* changes first and then [`Persist::commit`] can be used
-/// to write changes to disk.
-#[derive(Debug)]
-pub struct Persist<B, C> {
-    backend: B,
-    stage: C,
+/// Methods of this trait should not be called directly.
+pub trait PersistWith<Db>: Staged + Sized {
+    /// Parameters for [`PersistWith::create`].
+    type CreateParams;
+    /// Parameters for [`PersistWith::load`].
+    type LoadParams;
+    /// Error type of [`PersistWith::create`].
+    type CreateError;
+    /// Error type of [`PersistWith::load`].
+    type LoadError;
+    /// Error type of [`PersistWith::persist`].
+    type PersistError;
+
+    /// Initialize the `Db` and create `Self`.
+    fn create(db: &mut Db, params: Self::CreateParams) -> Result<Self, Self::CreateError>;
+
+    /// Initialize the `Db` and load a previously-persisted `Self`.
+    fn load(db: &mut Db, params: Self::LoadParams) -> Result<Option<Self>, Self::LoadError>;
+
+    /// Persist changes to the `Db`.
+    fn persist(
+        db: &mut Db,
+        changeset: &<Self as Staged>::ChangeSet,
+    ) -> Result<(), Self::PersistError>;
 }
 
-impl<B, C> Persist<B, C>
-where
-    B: PersistBackend<C>,
-    C: Default + Append,
-{
-    /// Create a new [`Persist`] from [`PersistBackend`].
-    pub fn new(backend: B) -> Self {
-        Self {
-            backend,
-            stage: Default::default(),
+type FutureResult<'a, T, E> = Pin<Box<dyn Future<Output = Result<T, E>> + Send + 'a>>;
+
+/// Trait that persists the type with an async `Db`.
+pub trait PersistAsyncWith<Db>: Staged + Sized {
+    /// Parameters for [`PersistAsyncWith::create`].
+    type CreateParams;
+    /// Parameters for [`PersistAsyncWith::load`].
+    type LoadParams;
+    /// Error type of [`PersistAsyncWith::create`].
+    type CreateError;
+    /// Error type of [`PersistAsyncWith::load`].
+    type LoadError;
+    /// Error type of [`PersistAsyncWith::persist`].
+    type PersistError;
+
+    /// Initialize the `Db` and create `Self`.
+    fn create(db: &mut Db, params: Self::CreateParams) -> FutureResult<Self, Self::CreateError>;
+
+    /// Initialize the `Db` and load a previously-persisted `Self`.
+    fn load(db: &mut Db, params: Self::LoadParams) -> FutureResult<Option<Self>, Self::LoadError>;
+
+    /// Persist changes to the `Db`.
+    fn persist<'a>(
+        db: &'a mut Db,
+        changeset: &'a <Self as Staged>::ChangeSet,
+    ) -> FutureResult<'a, (), Self::PersistError>;
+}
+
+/// Represents a persisted `T`.
+#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
+pub struct Persisted<T> {
+    inner: T,
+}
+
+impl<T> Persisted<T> {
+    /// Create a new persisted `T`.
+    pub fn create<Db>(db: &mut Db, params: T::CreateParams) -> Result<Self, T::CreateError>
+    where
+        T: PersistWith<Db>,
+    {
+        T::create(db, params).map(|inner| Self { inner })
+    }
+
+    /// Create a new persisted `T` with async `Db`.
+    pub async fn create_async<Db>(
+        db: &mut Db,
+        params: T::CreateParams,
+    ) -> Result<Self, T::CreateError>
+    where
+        T: PersistAsyncWith<Db>,
+    {
+        T::create(db, params).await.map(|inner| Self { inner })
+    }
+
+    /// Construct a persisted `T` from `Db`.
+    pub fn load<Db>(db: &mut Db, params: T::LoadParams) -> Result<Option<Self>, T::LoadError>
+    where
+        T: PersistWith<Db>,
+    {
+        Ok(T::load(db, params)?.map(|inner| Self { inner }))
+    }
+
+    /// Contruct a persisted `T` from an async `Db`.
+    pub async fn load_async<Db>(
+        db: &mut Db,
+        params: T::LoadParams,
+    ) -> Result<Option<Self>, T::LoadError>
+    where
+        T: PersistAsyncWith<Db>,
+    {
+        Ok(T::load(db, params).await?.map(|inner| Self { inner }))
+    }
+
+    /// Persist staged changes of `T` into `Db`.
+    ///
+    /// If the database errors, the staged changes will not be cleared.
+    pub fn persist<Db>(&mut self, db: &mut Db) -> Result<bool, T::PersistError>
+    where
+        T: PersistWith<Db>,
+    {
+        let stage = T::staged(&mut self.inner);
+        if stage.is_empty() {
+            return Ok(false);
         }
+        T::persist(db, &*stage)?;
+        stage.take();
+        Ok(true)
     }
 
-    /// Stage a `changeset` to be committed later with [`commit`].
+    /// Persist staged changes of `T` into an async `Db`.
     ///
-    /// [`commit`]: Self::commit
-    pub fn stage(&mut self, changeset: C) {
-        self.stage.append(changeset)
-    }
-
-    /// Get the changes that have not been committed yet.
-    pub fn staged(&self) -> &C {
-        &self.stage
-    }
-
-    /// Commit the staged changes to the underlying persistence backend.
-    ///
-    /// Changes that are committed (if any) are returned.
-    ///
-    /// # Error
-    ///
-    /// Returns a backend-defined error if this fails.
-    pub fn commit(&mut self) -> Result<Option<C>, B::WriteError> {
-        if self.stage.is_empty() {
-            return Ok(None);
+    /// If the database errors, the staged changes will not be cleared.
+    pub async fn persist_async<'a, Db>(
+        &'a mut self,
+        db: &'a mut Db,
+    ) -> Result<bool, T::PersistError>
+    where
+        T: PersistAsyncWith<Db>,
+    {
+        let stage = T::staged(&mut self.inner);
+        if stage.is_empty() {
+            return Ok(false);
         }
-        self.backend
-            .write_changes(&self.stage)
-            // if written successfully, take and return `self.stage`
-            .map(|_| Some(core::mem::take(&mut self.stage)))
-    }
-
-    /// Stages a new changeset and commits it (along with any other previously staged changes) to
-    /// the persistence backend
-    ///
-    /// Convenience method for calling [`stage`] and then [`commit`].
-    ///
-    /// [`stage`]: Self::stage
-    /// [`commit`]: Self::commit
-    pub fn stage_and_commit(&mut self, changeset: C) -> Result<Option<C>, B::WriteError> {
-        self.stage(changeset);
-        self.commit()
+        T::persist(db, &*stage).await?;
+        stage.take();
+        Ok(true)
     }
 }
 
-/// A persistence backend for [`Persist`].
-///
-/// `C` represents the changeset; a datatype that records changes made to in-memory data structures
-/// that are to be persisted, or retrieved from persistence.
-pub trait PersistBackend<C> {
-    /// The error the backend returns when it fails to write.
-    type WriteError: core::fmt::Debug;
+impl<T> Deref for Persisted<T> {
+    type Target = T;
 
-    /// The error the backend returns when it fails to load changesets `C`.
-    type LoadError: core::fmt::Debug;
-
-    /// Writes a changeset to the persistence backend.
-    ///
-    /// It is up to the backend what it does with this. It could store every changeset in a list or
-    /// it inserts the actual changes into a more structured database. All it needs to guarantee is
-    /// that [`load_from_persistence`] restores a keychain tracker to what it should be if all
-    /// changesets had been applied sequentially.
-    ///
-    /// [`load_from_persistence`]: Self::load_from_persistence
-    fn write_changes(&mut self, changeset: &C) -> Result<(), Self::WriteError>;
-
-    /// Return the aggregate changeset `C` from persistence.
-    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError>;
-}
-
-impl<C> PersistBackend<C> for () {
-    type WriteError = Infallible;
-
-    type LoadError = Infallible;
-
-    fn write_changes(&mut self, _changeset: &C) -> Result<(), Self::WriteError> {
-        Ok(())
-    }
-
-    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError> {
-        Ok(None)
+    fn deref(&self) -> &Self::Target {
+        &self.inner
+    }
+}
+
+impl<T> DerefMut for Persisted<T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.inner
     }
 }

crates/chain/src/rusqlite_impl.rs

@@ -0,0 +1,530 @@
+//! Module for stuff
+
+use crate::*;
+use core::str::FromStr;
+
+use alloc::{borrow::ToOwned, boxed::Box, string::ToString, sync::Arc, vec::Vec};
+use bitcoin::consensus::{Decodable, Encodable};
+use rusqlite;
+use rusqlite::named_params;
+use rusqlite::types::{FromSql, FromSqlError, FromSqlResult, ToSql, ToSqlOutput, ValueRef};
+use rusqlite::OptionalExtension;
+use rusqlite::Transaction;
+
+/// Table name for schemas.
+pub const SCHEMAS_TABLE_NAME: &str = "bdk_schemas";
+
+/// Initialize the schema table.
+fn init_schemas_table(db_tx: &Transaction) -> rusqlite::Result<()> {
+    let sql = format!("CREATE TABLE IF NOT EXISTS {}( name TEXT PRIMARY KEY NOT NULL, version INTEGER NOT NULL ) STRICT", SCHEMAS_TABLE_NAME);
+    db_tx.execute(&sql, ())?;
+    Ok(())
+}
+
+/// Get schema version of `schema_name`.
+fn schema_version(db_tx: &Transaction, schema_name: &str) -> rusqlite::Result<Option<u32>> {
+    let sql = format!(
+        "SELECT version FROM {} WHERE name=:name",
+        SCHEMAS_TABLE_NAME
+    );
+    db_tx
+        .query_row(&sql, named_params! { ":name": schema_name }, |row| {
+            row.get::<_, u32>("version")
+        })
+        .optional()
+}
+
+/// Set the `schema_version` of `schema_name`.
+fn set_schema_version(
+    db_tx: &Transaction,
+    schema_name: &str,
+    schema_version: u32,
+) -> rusqlite::Result<()> {
+    let sql = format!(
+        "REPLACE INTO {}(name, version) VALUES(:name, :version)",
+        SCHEMAS_TABLE_NAME,
+    );
+    db_tx.execute(
+        &sql,
+        named_params! { ":name": schema_name, ":version": schema_version },
+    )?;
+    Ok(())
+}
+
+/// Runs logic that initializes/migrates the table schemas.
+pub fn migrate_schema(
+    db_tx: &Transaction,
+    schema_name: &str,
+    versioned_scripts: &[&[&str]],
+) -> rusqlite::Result<()> {
+    init_schemas_table(db_tx)?;
+    let current_version = schema_version(db_tx, schema_name)?;
+    let exec_from = current_version.map_or(0_usize, |v| v as usize + 1);
+    let scripts_to_exec = versioned_scripts.iter().enumerate().skip(exec_from);
+    for (version, &script) in scripts_to_exec {
+        set_schema_version(db_tx, schema_name, version as u32)?;
+        for statement in script {
+            db_tx.execute(statement, ())?;
+        }
+    }
+    Ok(())
+}
+
+impl FromSql for Impl<bitcoin::Txid> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        bitcoin::Txid::from_str(value.as_str()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+impl ToSql for Impl<bitcoin::Txid> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.to_string().into())
+    }
+}
+
+impl FromSql for Impl<bitcoin::BlockHash> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        bitcoin::BlockHash::from_str(value.as_str()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+impl ToSql for Impl<bitcoin::BlockHash> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.to_string().into())
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl FromSql for Impl<DescriptorId> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        DescriptorId::from_str(value.as_str()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl ToSql for Impl<DescriptorId> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.to_string().into())
+    }
+}
+
+impl FromSql for Impl<bitcoin::Transaction> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        bitcoin::Transaction::consensus_decode_from_finite_reader(&mut value.as_bytes()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+impl ToSql for Impl<bitcoin::Transaction> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        let mut bytes = Vec::<u8>::new();
+        self.consensus_encode(&mut bytes).map_err(to_sql_error)?;
+        Ok(bytes.into())
+    }
+}
+
+impl FromSql for Impl<bitcoin::ScriptBuf> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        Ok(bitcoin::Script::from_bytes(value.as_bytes()?)
+            .to_owned()
+            .into())
+    }
+}
+
+impl ToSql for Impl<bitcoin::ScriptBuf> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.as_bytes().into())
+    }
+}
+
+impl FromSql for Impl<bitcoin::Amount> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        Ok(bitcoin::Amount::from_sat(value.as_i64()?.try_into().map_err(from_sql_error)?).into())
+    }
+}
+
+impl ToSql for Impl<bitcoin::Amount> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        let amount: i64 = self.to_sat().try_into().map_err(to_sql_error)?;
+        Ok(amount.into())
+    }
+}
+
+impl<A: Anchor + serde_crate::de::DeserializeOwned> FromSql for Impl<A> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        serde_json::from_str(value.as_str()?)
+            .map(Impl)
+            .map_err(from_sql_error)
+    }
+}
+
+impl<A: Anchor + serde_crate::Serialize> ToSql for Impl<A> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        serde_json::to_string(&self.0)
+            .map(Into::into)
+            .map_err(to_sql_error)
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl FromSql for Impl<miniscript::Descriptor<miniscript::DescriptorPublicKey>> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        miniscript::Descriptor::from_str(value.as_str()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl ToSql for Impl<miniscript::Descriptor<miniscript::DescriptorPublicKey>> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.to_string().into())
+    }
+}
+
+impl FromSql for Impl<bitcoin::Network> {
+    fn column_result(value: ValueRef<'_>) -> FromSqlResult<Self> {
+        bitcoin::Network::from_str(value.as_str()?)
+            .map(Self)
+            .map_err(from_sql_error)
+    }
+}
+
+impl ToSql for Impl<bitcoin::Network> {
+    fn to_sql(&self) -> rusqlite::Result<ToSqlOutput<'_>> {
+        Ok(self.to_string().into())
+    }
+}
+
+fn from_sql_error<E: std::error::Error + Send + Sync + 'static>(err: E) -> FromSqlError {
+    FromSqlError::Other(Box::new(err))
+}
+
+fn to_sql_error<E: std::error::Error + Send + Sync + 'static>(err: E) -> rusqlite::Error {
+    rusqlite::Error::ToSqlConversionFailure(Box::new(err))
+}
+
+impl<A> tx_graph::ChangeSet<A>
+where
+    A: Anchor + Clone + Ord + serde::Serialize + serde::de::DeserializeOwned,
+{
+    /// Schema name for [`tx_graph::ChangeSet`].
+    pub const SCHEMA_NAME: &'static str = "bdk_txgraph";
+    /// Name of table that stores full transactions and `last_seen` timestamps.
+    pub const TXS_TABLE_NAME: &'static str = "bdk_txs";
+    /// Name of table that stores floating txouts.
+    pub const TXOUTS_TABLE_NAME: &'static str = "bdk_txouts";
+    /// Name of table that stores [`Anchor`]s.
+    pub const ANCHORS_TABLE_NAME: &'static str = "bdk_anchors";
+
+    /// Initialize sqlite tables.
+    fn init_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        let schema_v0: &[&str] = &[
+            // full transactions
+            &format!(
+                "CREATE TABLE {} ( \
+                txid TEXT PRIMARY KEY NOT NULL, \
+                raw_tx BLOB, \
+                last_seen INTEGER \
+                ) STRICT",
+                Self::TXS_TABLE_NAME,
+            ),
+            // floating txouts
+            &format!(
+                "CREATE TABLE {} ( \
+                txid TEXT NOT NULL, \
+                vout INTEGER NOT NULL, \
+                value INTEGER NOT NULL, \
+                script BLOB NOT NULL, \
+                PRIMARY KEY (txid, vout) \
+                ) STRICT",
+                Self::TXOUTS_TABLE_NAME,
+            ),
+            // anchors
+            &format!(
+                "CREATE TABLE {} ( \
+                txid TEXT NOT NULL REFERENCES {} (txid), \
+                block_height INTEGER NOT NULL, \
+                block_hash TEXT NOT NULL, \
+                anchor BLOB NOT NULL, \
+                PRIMARY KEY (txid, block_height, block_hash) \
+                ) STRICT",
+                Self::ANCHORS_TABLE_NAME,
+                Self::TXS_TABLE_NAME,
+            ),
+        ];
+        migrate_schema(db_tx, Self::SCHEMA_NAME, &[schema_v0])
+    }
+
+    /// Construct a [`TxGraph`] from an sqlite database.
+    pub fn from_sqlite(db_tx: &rusqlite::Transaction) -> rusqlite::Result<Self> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut changeset = Self::default();
+
+        let mut statement = db_tx.prepare(&format!(
+            "SELECT txid, raw_tx, last_seen FROM {}",
+            Self::TXS_TABLE_NAME,
+        ))?;
+        let row_iter = statement.query_map([], |row| {
+            Ok((
+                row.get::<_, Impl<bitcoin::Txid>>("txid")?,
+                row.get::<_, Option<Impl<bitcoin::Transaction>>>("raw_tx")?,
+                row.get::<_, Option<u64>>("last_seen")?,
+            ))
+        })?;
+        for row in row_iter {
+            let (Impl(txid), tx, last_seen) = row?;
+            if let Some(Impl(tx)) = tx {
+                changeset.txs.insert(Arc::new(tx));
+            }
+            if let Some(last_seen) = last_seen {
+                changeset.last_seen.insert(txid, last_seen);
+            }
+        }
+
+        let mut statement = db_tx.prepare(&format!(
+            "SELECT txid, vout, value, script FROM {}",
+            Self::TXOUTS_TABLE_NAME,
+        ))?;
+        let row_iter = statement.query_map([], |row| {
+            Ok((
+                row.get::<_, Impl<bitcoin::Txid>>("txid")?,
+                row.get::<_, u32>("vout")?,
+                row.get::<_, Impl<bitcoin::Amount>>("value")?,
+                row.get::<_, Impl<bitcoin::ScriptBuf>>("script")?,
+            ))
+        })?;
+        for row in row_iter {
+            let (Impl(txid), vout, Impl(value), Impl(script_pubkey)) = row?;
+            changeset.txouts.insert(
+                bitcoin::OutPoint { txid, vout },
+                bitcoin::TxOut {
+                    value,
+                    script_pubkey,
+                },
+            );
+        }
+
+        let mut statement = db_tx.prepare(&format!(
+            "SELECT json(anchor), txid FROM {}",
+            Self::ANCHORS_TABLE_NAME,
+        ))?;
+        let row_iter = statement.query_map([], |row| {
+            Ok((
+                row.get::<_, Impl<A>>("json(anchor)")?,
+                row.get::<_, Impl<bitcoin::Txid>>("txid")?,
+            ))
+        })?;
+        for row in row_iter {
+            let (Impl(anchor), Impl(txid)) = row?;
+            changeset.anchors.insert((anchor, txid));
+        }
+
+        Ok(changeset)
+    }
+
+    /// Persist `changeset` to the sqlite database.
+    pub fn persist_to_sqlite(&self, db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(txid, raw_tx) VALUES(:txid, :raw_tx) ON CONFLICT(txid) DO UPDATE SET raw_tx=:raw_tx",
+            Self::TXS_TABLE_NAME,
+        ))?;
+        for tx in &self.txs {
+            statement.execute(named_params! {
+                ":txid": Impl(tx.compute_txid()),
+                ":raw_tx": Impl(tx.as_ref().clone()),
+            })?;
+        }
+
+        let mut statement = db_tx
+            .prepare_cached(&format!(
+                "INSERT INTO {}(txid, last_seen) VALUES(:txid, :last_seen) ON CONFLICT(txid) DO UPDATE SET last_seen=:last_seen",
+                Self::TXS_TABLE_NAME,
+            ))?;
+        for (&txid, &last_seen) in &self.last_seen {
+            statement.execute(named_params! {
+                ":txid": Impl(txid),
+                ":last_seen": Some(last_seen),
+            })?;
+        }
+
+        let mut statement = db_tx.prepare_cached(&format!(
+            "REPLACE INTO {}(txid, vout, value, script) VALUES(:txid, :vout, :value, :script)",
+            Self::TXOUTS_TABLE_NAME,
+        ))?;
+        for (op, txo) in &self.txouts {
+            statement.execute(named_params! {
+                ":txid": Impl(op.txid),
+                ":vout": op.vout,
+                ":value": Impl(txo.value),
+                ":script": Impl(txo.script_pubkey.clone()),
+            })?;
+        }
+
+        let mut statement = db_tx.prepare_cached(&format!(
+            "REPLACE INTO {}(txid, block_height, block_hash, anchor) VALUES(:txid, :block_height, :block_hash, jsonb(:anchor))",
+            Self::ANCHORS_TABLE_NAME,
+        ))?;
+        for (anchor, txid) in &self.anchors {
+            let anchor_block = anchor.anchor_block();
+            statement.execute(named_params! {
+                ":txid": Impl(*txid),
+                ":block_height": anchor_block.height,
+                ":block_hash": Impl(anchor_block.hash),
+                ":anchor": Impl(anchor.clone()),
+            })?;
+        }
+
+        Ok(())
+    }
+}
+
+impl local_chain::ChangeSet {
+    /// Schema name for the changeset.
+    pub const SCHEMA_NAME: &'static str = "bdk_localchain";
+    /// Name of sqlite table that stores blocks of [`LocalChain`](local_chain::LocalChain).
+    pub const BLOCKS_TABLE_NAME: &'static str = "bdk_blocks";
+
+    /// Initialize sqlite tables for persisting [`local_chain::LocalChain`].
+    fn init_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        let schema_v0: &[&str] = &[
+            // blocks
+            &format!(
+                "CREATE TABLE {} ( \
+                block_height INTEGER PRIMARY KEY NOT NULL, \
+                block_hash TEXT NOT NULL \
+                ) STRICT",
+                Self::BLOCKS_TABLE_NAME,
+            ),
+        ];
+        migrate_schema(db_tx, Self::SCHEMA_NAME, &[schema_v0])
+    }
+
+    /// Construct a [`LocalChain`](local_chain::LocalChain) from sqlite database.
+    pub fn from_sqlite(db_tx: &rusqlite::Transaction) -> rusqlite::Result<Self> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut changeset = Self::default();
+
+        let mut statement = db_tx.prepare(&format!(
+            "SELECT block_height, block_hash FROM {}",
+            Self::BLOCKS_TABLE_NAME,
+        ))?;
+        let row_iter = statement.query_map([], |row| {
+            Ok((
+                row.get::<_, u32>("block_height")?,
+                row.get::<_, Impl<bitcoin::BlockHash>>("block_hash")?,
+            ))
+        })?;
+        for row in row_iter {
+            let (height, Impl(hash)) = row?;
+            changeset.blocks.insert(height, Some(hash));
+        }
+
+        Ok(changeset)
+    }
+
+    /// Persist `changeset` to the sqlite database.
+    pub fn persist_to_sqlite(&self, db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut replace_statement = db_tx.prepare_cached(&format!(
+            "REPLACE INTO {}(block_height, block_hash) VALUES(:block_height, :block_hash)",
+            Self::BLOCKS_TABLE_NAME,
+        ))?;
+        let mut delete_statement = db_tx.prepare_cached(&format!(
+            "DELETE FROM {} WHERE block_height=:block_height",
+            Self::BLOCKS_TABLE_NAME,
+        ))?;
+        for (&height, &hash) in &self.blocks {
+            match hash {
+                Some(hash) => replace_statement.execute(named_params! {
+                    ":block_height": height,
+                    ":block_hash": Impl(hash),
+                })?,
+                None => delete_statement.execute(named_params! {
+                    ":block_height": height,
+                })?,
+            };
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl keychain_txout::ChangeSet {
+    /// Schema name for the changeset.
+    pub const SCHEMA_NAME: &'static str = "bdk_keychaintxout";
+    /// Name for table that stores last revealed indices per descriptor id.
+    pub const LAST_REVEALED_TABLE_NAME: &'static str = "bdk_descriptor_last_revealed";
+
+    /// Initialize sqlite tables for persisting
+    /// [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex).
+    fn init_sqlite_tables(db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        let schema_v0: &[&str] = &[
+            // last revealed
+            &format!(
+                "CREATE TABLE {} ( \
+                descriptor_id TEXT PRIMARY KEY NOT NULL, \
+                last_revealed INTEGER NOT NULL \
+                ) STRICT",
+                Self::LAST_REVEALED_TABLE_NAME,
+            ),
+        ];
+        migrate_schema(db_tx, Self::SCHEMA_NAME, &[schema_v0])
+    }
+
+    /// Construct [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex) from sqlite database
+    /// and given parameters.
+    pub fn from_sqlite(db_tx: &rusqlite::Transaction) -> rusqlite::Result<Self> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut changeset = Self::default();
+
+        let mut statement = db_tx.prepare(&format!(
+            "SELECT descriptor_id, last_revealed FROM {}",
+            Self::LAST_REVEALED_TABLE_NAME,
+        ))?;
+        let row_iter = statement.query_map([], |row| {
+            Ok((
+                row.get::<_, Impl<DescriptorId>>("descriptor_id")?,
+                row.get::<_, u32>("last_revealed")?,
+            ))
+        })?;
+        for row in row_iter {
+            let (Impl(descriptor_id), last_revealed) = row?;
+            changeset.last_revealed.insert(descriptor_id, last_revealed);
+        }
+
+        Ok(changeset)
+    }
+
+    /// Persist `changeset` to the sqlite database.
+    pub fn persist_to_sqlite(&self, db_tx: &rusqlite::Transaction) -> rusqlite::Result<()> {
+        Self::init_sqlite_tables(db_tx)?;
+
+        let mut statement = db_tx.prepare_cached(&format!(
+            "REPLACE INTO {}(descriptor_id, last_revealed) VALUES(:descriptor_id, :last_revealed)",
+            Self::LAST_REVEALED_TABLE_NAME,
+        ))?;
+        for (&descriptor_id, &last_revealed) in &self.last_revealed {
+            statement.execute(named_params! {
+                ":descriptor_id": Impl(descriptor_id),
+                ":last_revealed": last_revealed,
+            })?;
+        }
+
+        Ok(())
+    }
+}

crates/chain/src/spk_client.rs

@@ -0,0 +1,388 @@
+//! Helper types for spk-based blockchain clients.
+
+use crate::{
+    collections::BTreeMap, local_chain::CheckPoint, ConfirmationBlockTime, Indexed, TxGraph,
+};
+use alloc::boxed::Box;
+use bitcoin::{OutPoint, Script, ScriptBuf, Txid};
+use core::marker::PhantomData;
+
+/// Data required to perform a spk-based blockchain client sync.
+///
+/// A client sync fetches relevant chain data for a known list of scripts, transaction ids and
+/// outpoints. The sync process also updates the chain from the given [`CheckPoint`].
+pub struct SyncRequest {
+    /// A checkpoint for the current chain [`LocalChain::tip`].
+    /// The sync process will return a new chain update that extends this tip.
+    ///
+    /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
+    pub chain_tip: CheckPoint,
+    /// Transactions that spend from or to these indexed script pubkeys.
+    pub spks: Box<dyn ExactSizeIterator<Item = ScriptBuf> + Send>,
+    /// Transactions with these txids.
+    pub txids: Box<dyn ExactSizeIterator<Item = Txid> + Send>,
+    /// Transactions with these outpoints or spent from these outpoints.
+    pub outpoints: Box<dyn ExactSizeIterator<Item = OutPoint> + Send>,
+}
+
+impl SyncRequest {
+    /// Construct a new [`SyncRequest`] from a given `cp` tip.
+    pub fn from_chain_tip(cp: CheckPoint) -> Self {
+        Self {
+            chain_tip: cp,
+            spks: Box::new(core::iter::empty()),
+            txids: Box::new(core::iter::empty()),
+            outpoints: Box::new(core::iter::empty()),
+        }
+    }
+
+    /// Set the [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_spks(
+        mut self,
+        spks: impl IntoIterator<IntoIter = impl ExactSizeIterator<Item = ScriptBuf> + Send + 'static>,
+    ) -> Self {
+        self.spks = Box::new(spks.into_iter());
+        self
+    }
+
+    /// Set the [`Txid`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_txids(
+        mut self,
+        txids: impl IntoIterator<IntoIter = impl ExactSizeIterator<Item = Txid> + Send + 'static>,
+    ) -> Self {
+        self.txids = Box::new(txids.into_iter());
+        self
+    }
+
+    /// Set the [`OutPoint`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_outpoints(
+        mut self,
+        outpoints: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = OutPoint> + Send + 'static,
+        >,
+    ) -> Self {
+        self.outpoints = Box::new(outpoints.into_iter());
+        self
+    }
+
+    /// Chain on additional [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_spks(
+        mut self,
+        spks: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = ScriptBuf> + Send + 'static,
+            Item = ScriptBuf,
+        >,
+    ) -> Self {
+        self.spks = Box::new(ExactSizeChain::new(self.spks, spks.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`Txid`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_txids(
+        mut self,
+        txids: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = Txid> + Send + 'static,
+            Item = Txid,
+        >,
+    ) -> Self {
+        self.txids = Box::new(ExactSizeChain::new(self.txids, txids.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`OutPoint`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_outpoints(
+        mut self,
+        outpoints: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = OutPoint> + Send + 'static,
+            Item = OutPoint,
+        >,
+    ) -> Self {
+        self.outpoints = Box::new(ExactSizeChain::new(self.outpoints, outpoints.into_iter()));
+        self
+    }
+
+    /// Add a closure that will be called for [`Script`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks(
+        mut self,
+        mut inspect: impl FnMut(&Script) + Send + Sync + 'static,
+    ) -> Self {
+        self.spks = Box::new(self.spks.inspect(move |spk| inspect(spk)));
+        self
+    }
+
+    /// Add a closure that will be called for [`Txid`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_txids(mut self, mut inspect: impl FnMut(&Txid) + Send + Sync + 'static) -> Self {
+        self.txids = Box::new(self.txids.inspect(move |txid| inspect(txid)));
+        self
+    }
+
+    /// Add a closure that will be called for [`OutPoint`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_outpoints(
+        mut self,
+        mut inspect: impl FnMut(&OutPoint) + Send + Sync + 'static,
+    ) -> Self {
+        self.outpoints = Box::new(self.outpoints.inspect(move |op| inspect(op)));
+        self
+    }
+
+    /// Populate the request with revealed script pubkeys from `index` with the given `spk_range`.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[cfg(feature = "miniscript")]
+    #[must_use]
+    pub fn populate_with_revealed_spks<K: Clone + Ord + core::fmt::Debug + Send + Sync>(
+        self,
+        index: &crate::indexer::keychain_txout::KeychainTxOutIndex<K>,
+        spk_range: impl core::ops::RangeBounds<K>,
+    ) -> Self {
+        use alloc::borrow::ToOwned;
+        use alloc::vec::Vec;
+        self.chain_spks(
+            index
+                .revealed_spks(spk_range)
+                .map(|(_, spk)| spk.to_owned())
+                .collect::<Vec<_>>(),
+        )
+    }
+}
+
+/// Data returned from a spk-based blockchain client sync.
+///
+/// See also [`SyncRequest`].
+pub struct SyncResult<A = ConfirmationBlockTime> {
+    /// The update to apply to the receiving [`TxGraph`].
+    pub graph_update: TxGraph<A>,
+    /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain).
+    pub chain_update: CheckPoint,
+}
+
+/// Data required to perform a spk-based blockchain client full scan.
+///
+/// A client full scan iterates through all the scripts for the given keychains, fetching relevant
+/// data until some stop gap number of scripts is found that have no data. This operation is
+/// generally only used when importing or restoring previously used keychains in which the list of
+/// used scripts is not known. The full scan process also updates the chain from the given [`CheckPoint`].
+pub struct FullScanRequest<K> {
+    /// A checkpoint for the current [`LocalChain::tip`].
+    /// The full scan process will return a new chain update that extends this tip.
+    ///
+    /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
+    pub chain_tip: CheckPoint,
+    /// Iterators of script pubkeys indexed by the keychain index.
+    pub spks_by_keychain: BTreeMap<K, Box<dyn Iterator<Item = Indexed<ScriptBuf>> + Send>>,
+}
+
+impl<K: Ord + Clone> FullScanRequest<K> {
+    /// Construct a new [`FullScanRequest`] from a given `chain_tip`.
+    #[must_use]
+    pub fn from_chain_tip(chain_tip: CheckPoint) -> Self {
+        Self {
+            chain_tip,
+            spks_by_keychain: BTreeMap::new(),
+        }
+    }
+
+    /// Construct a new [`FullScanRequest`] from a given `chain_tip` and `index`.
+    ///
+    /// Unbounded script pubkey iterators for each keychain (`K`) are extracted using
+    /// [`KeychainTxOutIndex::all_unbounded_spk_iters`] and is used to populate the
+    /// [`FullScanRequest`].
+    ///
+    /// [`KeychainTxOutIndex::all_unbounded_spk_iters`]: crate::indexer::keychain_txout::KeychainTxOutIndex::all_unbounded_spk_iters
+    #[cfg(feature = "miniscript")]
+    #[must_use]
+    pub fn from_keychain_txout_index(
+        chain_tip: CheckPoint,
+        index: &crate::indexer::keychain_txout::KeychainTxOutIndex<K>,
+    ) -> Self
+    where
+        K: core::fmt::Debug,
+    {
+        let mut req = Self::from_chain_tip(chain_tip);
+        for (keychain, spks) in index.all_unbounded_spk_iters() {
+            req = req.set_spks_for_keychain(keychain, spks);
+        }
+        req
+    }
+
+    /// Set the [`Script`]s for a given `keychain`.
+    ///
+    /// This consumes the [`FullScanRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_spks_for_keychain(
+        mut self,
+        keychain: K,
+        spks: impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send + 'static>,
+    ) -> Self {
+        self.spks_by_keychain
+            .insert(keychain, Box::new(spks.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`FullScanRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_spks_for_keychain(
+        mut self,
+        keychain: K,
+        spks: impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send + 'static>,
+    ) -> Self {
+        match self.spks_by_keychain.remove(&keychain) {
+            // clippy here suggests to remove `into_iter` from `spks.into_iter()`, but doing so
+            // results in a compilation error
+            #[allow(clippy::useless_conversion)]
+            Some(keychain_spks) => self
+                .spks_by_keychain
+                .insert(keychain, Box::new(keychain_spks.chain(spks.into_iter()))),
+            None => self
+                .spks_by_keychain
+                .insert(keychain, Box::new(spks.into_iter())),
+        };
+        self
+    }
+
+    /// Add a closure that will be called for every [`Script`] previously added to any keychain in
+    /// this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks_for_all_keychains(
+        mut self,
+        inspect: impl FnMut(K, u32, &Script) + Send + Sync + Clone + 'static,
+    ) -> Self
+    where
+        K: Send + 'static,
+    {
+        for (keychain, spks) in core::mem::take(&mut self.spks_by_keychain) {
+            let mut inspect = inspect.clone();
+            self.spks_by_keychain.insert(
+                keychain.clone(),
+                Box::new(spks.inspect(move |(i, spk)| inspect(keychain.clone(), *i, spk))),
+            );
+        }
+        self
+    }
+
+    /// Add a closure that will be called for every [`Script`] previously added to a given
+    /// `keychain` in this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks_for_keychain(
+        mut self,
+        keychain: K,
+        mut inspect: impl FnMut(u32, &Script) + Send + Sync + 'static,
+    ) -> Self
+    where
+        K: Send + 'static,
+    {
+        if let Some(spks) = self.spks_by_keychain.remove(&keychain) {
+            self.spks_by_keychain.insert(
+                keychain,
+                Box::new(spks.inspect(move |(i, spk)| inspect(*i, spk))),
+            );
+        }
+        self
+    }
+}
+
+/// Data returned from a spk-based blockchain client full scan.
+///
+/// See also [`FullScanRequest`].
+pub struct FullScanResult<K, A = ConfirmationBlockTime> {
+    /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain).
+    pub graph_update: TxGraph<A>,
+    /// The update to apply to the receiving [`TxGraph`].
+    pub chain_update: CheckPoint,
+    /// Last active indices for the corresponding keychains (`K`).
+    pub last_active_indices: BTreeMap<K, u32>,
+}
+
+/// A version of [`core::iter::Chain`] which can combine two [`ExactSizeIterator`]s to form a new
+/// [`ExactSizeIterator`].
+///
+/// The danger of this is explained in [the `ExactSizeIterator` docs]
+/// (https://doc.rust-lang.org/core/iter/trait.ExactSizeIterator.html#when-shouldnt-an-adapter-be-exactsizeiterator).
+/// This does not apply here since it would be impossible to scan an item count that overflows
+/// `usize` anyway.
+struct ExactSizeChain<A, B, I> {
+    a: Option<A>,
+    b: Option<B>,
+    i: PhantomData<I>,
+}
+
+impl<A, B, I> ExactSizeChain<A, B, I> {
+    fn new(a: A, b: B) -> Self {
+        ExactSizeChain {
+            a: Some(a),
+            b: Some(b),
+            i: PhantomData,
+        }
+    }
+}
+
+impl<A, B, I> Iterator for ExactSizeChain<A, B, I>
+where
+    A: Iterator<Item = I>,
+    B: Iterator<Item = I>,
+{
+    type Item = I;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(a) = &mut self.a {
+            let item = a.next();
+            if item.is_some() {
+                return item;
+            }
+            self.a = None;
+        }
+        if let Some(b) = &mut self.b {
+            let item = b.next();
+            if item.is_some() {
+                return item;
+            }
+            self.b = None;
+        }
+        None
+    }
+}
+
+impl<A, B, I> ExactSizeIterator for ExactSizeChain<A, B, I>
+where
+    A: ExactSizeIterator<Item = I>,
+    B: ExactSizeIterator<Item = I>,
+{
+    fn len(&self) -> usize {
+        let a_len = self.a.as_ref().map(|a| a.len()).unwrap_or(0);
+        let b_len = self.b.as_ref().map(|a| a.len()).unwrap_or(0);
+        a_len + b_len
+    }
+}

crates/chain/src/spk_iter.rs

@@ -1,6 +1,7 @@
 use crate::{
     bitcoin::{secp256k1::Secp256k1, ScriptBuf},
     miniscript::{Descriptor, DescriptorPublicKey},
+    Indexed,
 };
 use core::{borrow::Borrow, ops::Bound, ops::RangeBounds};
 
@@ -97,7 +98,7 @@ impl<D> Iterator for SpkIterator<D>
 where
     D: Borrow<Descriptor<DescriptorPublicKey>>,
 {
-    type Item = (u32, ScriptBuf);
+    type Item = Indexed<ScriptBuf>;
 
     fn next(&mut self) -> Option<Self::Item> {
         // For non-wildcard descriptors, we expect the first element to be Some((0, spk)), then None after.
@@ -136,7 +137,7 @@ where
 mod test {
     use crate::{
         bitcoin::secp256k1::Secp256k1,
-        keychain::KeychainTxOutIndex,
+        indexer::keychain_txout::KeychainTxOutIndex,
         miniscript::{Descriptor, DescriptorPublicKey},
         spk_iter::{SpkIterator, BIP32_MAX_INDEX},
     };
@@ -158,8 +159,12 @@ mod test {
         let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
         let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
 
-        txout_index.add_keychain(TestKeychain::External, external_descriptor.clone());
-        txout_index.add_keychain(TestKeychain::Internal, internal_descriptor.clone());
+        let _ = txout_index
+            .insert_descriptor(TestKeychain::External, external_descriptor.clone())
+            .unwrap();
+        let _ = txout_index
+            .insert_descriptor(TestKeychain::Internal, internal_descriptor.clone())
+            .unwrap();
 
         (txout_index, external_descriptor, internal_descriptor)
     }
@@ -258,17 +263,10 @@ mod test {
             None
         );
     }
-
-    // The following dummy traits were created to test if SpkIterator is working properly.
-    trait TestSendStatic: Send + 'static {
-        fn test(&self) -> u32 {
-            20
-        }
-    }
-
-    impl TestSendStatic for SpkIterator<Descriptor<DescriptorPublicKey>> {
-        fn test(&self) -> u32 {
-            20
-        }
-    }
+}
+
+#[test]
+fn spk_iterator_is_send_and_static() {
+    fn is_send_and_static<A: Send + 'static>() {}
+    is_send_and_static::<SpkIterator<Descriptor<DescriptorPublicKey>>>()
 }

crates/chain/src/tx_data_traits.rs

@@ -20,8 +20,7 @@ use alloc::vec::Vec;
 /// # use bdk_chain::local_chain::LocalChain;
 /// # use bdk_chain::tx_graph::TxGraph;
 /// # use bdk_chain::BlockId;
-/// # use bdk_chain::ConfirmationHeightAnchor;
-/// # use bdk_chain::ConfirmationTimeHeightAnchor;
+/// # use bdk_chain::ConfirmationBlockTime;
 /// # use bdk_chain::example_utils::*;
 /// # use bitcoin::hashes::Hash;
 /// // Initialize the local chain with two blocks.
@@ -43,46 +42,26 @@ use alloc::vec::Vec;
 /// let mut graph_a = TxGraph::<BlockId>::default();
 /// let _ = graph_a.insert_tx(tx.clone());
 /// graph_a.insert_anchor(
-///     tx.txid(),
+///     tx.compute_txid(),
 ///     BlockId {
 ///         height: 1,
 ///         hash: Hash::hash("first".as_bytes()),
 ///     },
 /// );
 ///
-/// // Insert `tx` into a `TxGraph` that uses `ConfirmationHeightAnchor` as the anchor type.
-/// // This anchor records the anchor block and the confirmation height of the transaction.
-/// // When a transaction is anchored with `ConfirmationHeightAnchor`, the anchor block and
-/// // confirmation block can be different. However, the confirmation block cannot be higher than
-/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
-/// let mut graph_b = TxGraph::<ConfirmationHeightAnchor>::default();
-/// let _ = graph_b.insert_tx(tx.clone());
-/// graph_b.insert_anchor(
-///     tx.txid(),
-///     ConfirmationHeightAnchor {
-///         anchor_block: BlockId {
-///             height: 2,
-///             hash: Hash::hash("second".as_bytes()),
-///         },
-///         confirmation_height: 1,
-///     },
-/// );
-///
-/// // Insert `tx` into a `TxGraph` that uses `ConfirmationTimeHeightAnchor` as the anchor type.
-/// // This anchor records the anchor block, the confirmation height and time of the transaction.
-/// // When a transaction is anchored with `ConfirmationTimeHeightAnchor`, the anchor block and
-/// // confirmation block can be different. However, the confirmation block cannot be higher than
-/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
-/// let mut graph_c = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+/// // Insert `tx` into a `TxGraph` that uses `ConfirmationBlockTime` as the anchor type.
+/// // This anchor records the anchor block and the confirmation time of the transaction. When a
+/// // transaction is anchored with `ConfirmationBlockTime`, the anchor block and confirmation block
+/// // of the transaction is the same block.
+/// let mut graph_c = TxGraph::<ConfirmationBlockTime>::default();
 /// let _ = graph_c.insert_tx(tx.clone());
 /// graph_c.insert_anchor(
-///     tx.txid(),
-///     ConfirmationTimeHeightAnchor {
-///         anchor_block: BlockId {
+///     tx.compute_txid(),
+///     ConfirmationBlockTime {
+///         block_id: BlockId {
 ///             height: 2,
 ///             hash: Hash::hash("third".as_bytes()),
 ///         },
-///         confirmation_height: 1,
 ///         confirmation_time: 123,
 ///     },
 /// );
@@ -113,17 +92,26 @@ pub trait AnchorFromBlockPosition: Anchor {
     fn from_block_position(block: &bitcoin::Block, block_id: BlockId, tx_pos: usize) -> Self;
 }
 
-/// Trait that makes an object appendable.
-pub trait Append {
-    /// Append another object of the same type onto `self`.
-    fn append(&mut self, other: Self);
+/// Trait that makes an object mergeable.
+pub trait Merge: Default {
+    /// Merge another object of the same type onto `self`.
+    fn merge(&mut self, other: Self);
 
     /// Returns whether the structure is considered empty.
     fn is_empty(&self) -> bool;
+
+    /// Take the value, replacing it with the default value.
+    fn take(&mut self) -> Option<Self> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(core::mem::take(self))
+        }
+    }
 }
 
-impl<K: Ord, V> Append for BTreeMap<K, V> {
-    fn append(&mut self, other: Self) {
+impl<K: Ord, V> Merge for BTreeMap<K, V> {
+    fn merge(&mut self, other: Self) {
         // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
         // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
         BTreeMap::extend(self, other)
@@ -134,8 +122,8 @@ impl<K: Ord, V> Append for BTreeMap<K, V> {
     }
 }
 
-impl<T: Ord> Append for BTreeSet<T> {
-    fn append(&mut self, other: Self) {
+impl<T: Ord> Merge for BTreeSet<T> {
+    fn merge(&mut self, other: Self) {
         // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
         // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
         BTreeSet::extend(self, other)
@@ -146,8 +134,8 @@ impl<T: Ord> Append for BTreeSet<T> {
     }
 }
 
-impl<T> Append for Vec<T> {
-    fn append(&mut self, mut other: Self) {
+impl<T> Merge for Vec<T> {
+    fn merge(&mut self, mut other: Self) {
         Vec::append(self, &mut other)
     }
 
@@ -156,30 +144,30 @@ impl<T> Append for Vec<T> {
     }
 }
 
-macro_rules! impl_append_for_tuple {
+macro_rules! impl_merge_for_tuple {
     ($($a:ident $b:tt)*) => {
-        impl<$($a),*> Append for ($($a,)*) where $($a: Append),* {
+        impl<$($a),*> Merge for ($($a,)*) where $($a: Merge),* {
 
-            fn append(&mut self, _other: Self) {
-                $(Append::append(&mut self.$b, _other.$b) );*
+            fn merge(&mut self, _other: Self) {
+                $(Merge::merge(&mut self.$b, _other.$b) );*
             }
 
             fn is_empty(&self) -> bool {
-                $(Append::is_empty(&self.$b) && )* true
+                $(Merge::is_empty(&self.$b) && )* true
             }
         }
     }
 }
 
-impl_append_for_tuple!();
-impl_append_for_tuple!(T0 0);
-impl_append_for_tuple!(T0 0 T1 1);
-impl_append_for_tuple!(T0 0 T1 1 T2 2);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9);
-impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9 T10 10);
+impl_merge_for_tuple!();
+impl_merge_for_tuple!(T0 0);
+impl_merge_for_tuple!(T0 0 T1 1);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9);
+impl_merge_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9 T10 10);

crates/chain/tests/common/mod.rs

@@ -1,3 +1,5 @@
+#![cfg(feature = "miniscript")]
+
 mod tx_template;
 #[allow(unused_imports)]
 pub use tx_template::*;
@@ -32,12 +34,9 @@ macro_rules! local_chain {
 macro_rules! chain_update {
     [ $(($height:expr, $hash:expr)), * ] => {{
         #[allow(unused_mut)]
-        bdk_chain::local_chain::Update {
-            tip: bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect())
-                .expect("chain must have genesis block")
-                .tip(),
-            introduce_older_blocks: true,
-        }
+        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect())
+            .expect("chain must have genesis block")
+            .tip()
     }};
 }
 
@@ -70,9 +69,21 @@ macro_rules! changeset {
 #[allow(unused)]
 pub fn new_tx(lt: u32) -> bitcoin::Transaction {
     bitcoin::Transaction {
-        version: 0x00,
+        version: bitcoin::transaction::Version::non_standard(0x00),
         lock_time: bitcoin::absolute::LockTime::from_consensus(lt),
         input: vec![],
         output: vec![],
     }
 }
+
+#[allow(unused)]
+pub const DESCRIPTORS: [&str; 7] = [
+    "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)",
+    "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)",
+    "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0/*)",
+    "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)",
+    "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)",
+    "wpkh(xprv9s21ZrQH143K4EXURwMHuLS469fFzZyXk7UUpdKfQwhoHcAiYTakpe8pMU2RiEdvrU9McyuE7YDoKcXkoAwEGoK53WBDnKKv2zZbb9BzttX/1/0/*)",
+    // non-wildcard
+    "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)",
+];

crates/chain/tests/common/tx_template.rs

@@ -1,10 +1,12 @@
+#![cfg(feature = "miniscript")]
+
 use rand::distributions::{Alphanumeric, DistString};
 use std::collections::HashMap;
 
-use bdk_chain::{tx_graph::TxGraph, BlockId, SpkTxOutIndex};
+use bdk_chain::{spk_txout::SpkTxOutIndex, tx_graph::TxGraph, Anchor};
 use bitcoin::{
-    locktime::absolute::LockTime, secp256k1::Secp256k1, OutPoint, ScriptBuf, Sequence, Transaction,
-    TxIn, TxOut, Txid, Witness,
+    locktime::absolute::LockTime, secp256k1::Secp256k1, transaction, Amount, OutPoint, ScriptBuf,
+    Sequence, Transaction, TxIn, TxOut, Txid, Witness,
 };
 use miniscript::Descriptor;
 
@@ -49,11 +51,12 @@ impl TxOutTemplate {
 }
 
 #[allow(dead_code)]
-pub fn init_graph<'a>(
-    tx_templates: impl IntoIterator<Item = &'a TxTemplate<'a, BlockId>>,
-) -> (TxGraph<BlockId>, SpkTxOutIndex<u32>, HashMap<&'a str, Txid>) {
-    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
-    let mut graph = TxGraph::<BlockId>::default();
+pub fn init_graph<'a, A: Anchor + Clone + 'a>(
+    tx_templates: impl IntoIterator<Item = &'a TxTemplate<'a, A>>,
+) -> (TxGraph<A>, SpkTxOutIndex<u32>, HashMap<&'a str, Txid>) {
+    let (descriptor, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), super::DESCRIPTORS[2]).unwrap();
+    let mut graph = TxGraph::<A>::default();
     let mut spk_index = SpkTxOutIndex::default();
     (0..10).for_each(|index| {
         spk_index.insert_spk(
@@ -68,7 +71,7 @@ pub fn init_graph<'a>(
 
     for (bogus_txin_vout, tx_tmp) in tx_templates.into_iter().enumerate() {
         let tx = Transaction {
-            version: 0,
+            version: transaction::Version::non_standard(0),
             lock_time: LockTime::ZERO,
             input: tx_tmp
                 .inputs
@@ -111,26 +114,24 @@ pub fn init_graph<'a>(
                 .iter()
                 .map(|output| match &output.spk_index {
                     None => TxOut {
-                        value: output.value,
+                        value: Amount::from_sat(output.value),
                         script_pubkey: ScriptBuf::new(),
                     },
                     Some(index) => TxOut {
-                        value: output.value,
-                        script_pubkey: spk_index.spk_at_index(index).unwrap().to_owned(),
+                        value: Amount::from_sat(output.value),
+                        script_pubkey: spk_index.spk_at_index(index).unwrap(),
                     },
                 })
                 .collect(),
         };
 
-        tx_ids.insert(tx_tmp.tx_name, tx.txid());
+        tx_ids.insert(tx_tmp.tx_name, tx.compute_txid());
         spk_index.scan(&tx);
         let _ = graph.insert_tx(tx.clone());
         for anchor in tx_tmp.anchors.iter() {
-            let _ = graph.insert_anchor(tx.txid(), *anchor);
-        }
-        if let Some(seen_at) = tx_tmp.last_seen {
-            let _ = graph.insert_seen_at(tx.txid(), seen_at);
+            let _ = graph.insert_anchor(tx.compute_txid(), anchor.clone());
         }
+        let _ = graph.insert_seen_at(tx.compute_txid(), tx_tmp.last_seen.unwrap_or(0));
     }
     (graph, spk_index, tx_ids)
 }

crates/chain/tests/test_indexed_tx_graph.rs

@@ -1,15 +1,18 @@
+#![cfg(feature = "miniscript")]
+
 #[macro_use]
 mod common;
 
-use std::collections::BTreeSet;
+use std::{collections::BTreeSet, sync::Arc};
 
+use crate::common::DESCRIPTORS;
 use bdk_chain::{
     indexed_tx_graph::{self, IndexedTxGraph},
-    keychain::{self, Balance, KeychainTxOutIndex},
+    indexer::keychain_txout::KeychainTxOutIndex,
     local_chain::LocalChain,
-    tx_graph, BlockId, ChainPosition, ConfirmationHeightAnchor,
+    tx_graph, Balance, ChainPosition, ConfirmationBlockTime, DescriptorExt,
 };
-use bitcoin::{secp256k1::Secp256k1, OutPoint, Script, ScriptBuf, Transaction, TxIn, TxOut};
+use bitcoin::{secp256k1::Secp256k1, Amount, OutPoint, ScriptBuf, Transaction, TxIn, TxOut};
 use miniscript::Descriptor;
 
 /// Ensure [`IndexedTxGraph::insert_relevant_txs`] can successfully index transactions NOT presented
@@ -21,25 +24,28 @@ use miniscript::Descriptor;
 /// agnostic.
 #[test]
 fn insert_relevant_txs() {
-    const DESCRIPTOR: &str = "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)";
-    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTOR)
+    use bdk_chain::indexer::keychain_txout;
+    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTORS[0])
         .expect("must be valid");
     let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
     let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::new(
+    let mut graph = IndexedTxGraph::<ConfirmationBlockTime, KeychainTxOutIndex<()>>::new(
         KeychainTxOutIndex::new(10),
     );
-    graph.index.add_keychain((), descriptor);
+    let _ = graph
+        .index
+        .insert_descriptor((), descriptor.clone())
+        .unwrap();
 
     let tx_a = Transaction {
         output: vec![
             TxOut {
-                value: 10_000,
+                value: Amount::from_sat(10_000),
                 script_pubkey: spk_0,
             },
             TxOut {
-                value: 20_000,
+                value: Amount::from_sat(20_000),
                 script_pubkey: spk_1,
             },
         ],
@@ -48,7 +54,7 @@ fn insert_relevant_txs() {
 
     let tx_b = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx_a.txid(), 0),
+            previous_output: OutPoint::new(tx_a.compute_txid(), 0),
             ..Default::default()
         }],
         ..common::new_tx(1)
@@ -56,7 +62,7 @@ fn insert_relevant_txs() {
 
     let tx_c = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx_a.txid(), 1),
+            previous_output: OutPoint::new(tx_a.compute_txid(), 1),
             ..Default::default()
         }],
         ..common::new_tx(2)
@@ -65,11 +71,13 @@ fn insert_relevant_txs() {
     let txs = [tx_c, tx_b, tx_a];
 
     let changeset = indexed_tx_graph::ChangeSet {
-        graph: tx_graph::ChangeSet {
-            txs: txs.clone().into(),
+        tx_graph: tx_graph::ChangeSet {
+            txs: txs.iter().cloned().map(Arc::new).collect(),
             ..Default::default()
         },
-        indexer: keychain::ChangeSet([((), 9_u32)].into()),
+        indexer: keychain_txout::ChangeSet {
+            last_revealed: [(descriptor.descriptor_id(), 9_u32)].into(),
+        },
     };
 
     assert_eq!(
@@ -77,10 +85,17 @@ fn insert_relevant_txs() {
         changeset,
     );
 
-    assert_eq!(graph.initial_changeset(), changeset,);
+    // The initial changeset will also contain info about the keychain we added
+    let initial_changeset = indexed_tx_graph::ChangeSet {
+        tx_graph: changeset.tx_graph,
+        indexer: keychain_txout::ChangeSet {
+            last_revealed: changeset.indexer.last_revealed,
+        },
+    };
+
+    assert_eq!(graph.initial_changeset(), initial_changeset);
 }
 
-#[test]
 /// Ensure consistency IndexedTxGraph list_* and balance methods. These methods lists
 /// relevant txouts and utxos from the information fetched from a ChainOracle (here a LocalChain).
 ///
@@ -98,8 +113,8 @@ fn insert_relevant_txs() {
 /// tx1: A Coinbase, sending 70000 sats to "trusted" address. [Block 0]
 /// tx2: A external Receive, sending 30000 sats to "untrusted" address. [Block 1]
 /// tx3: Internal Spend. Spends tx2 and returns change of 10000 to "trusted" address. [Block 2]
-/// tx4: Mempool tx, sending 20000 sats to "trusted" address.
-/// tx5: Mempool tx, sending 15000 sats to "untested" address.
+/// tx4: Mempool tx, sending 20000 sats to "untrusted" address.
+/// tx5: Mempool tx, sending 15000 sats to "trusted" address.
 /// tx6: Complete unrelated tx. [Block 3]
 ///
 /// Different transactions are added via `insert_relevant_txs`.
@@ -108,7 +123,7 @@ fn insert_relevant_txs() {
 ///
 /// Finally Add more blocks to local chain until tx1 coinbase maturity hits.
 /// Assert maturity at coinbase maturity inflection height. Block height 98 and 99.
-
+#[test]
 fn test_list_owned_txouts() {
     // Create Local chains
     let local_chain = LocalChain::from_blocks((0..150).map(|i| (i as u32, h!("random"))).collect())
@@ -116,15 +131,23 @@ fn test_list_owned_txouts() {
 
     // Initiate IndexedTxGraph
 
-    let (desc_1, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
-    let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)").unwrap();
+    let (desc_1, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), common::DESCRIPTORS[2]).unwrap();
+    let (desc_2, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), common::DESCRIPTORS[3]).unwrap();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::new(
+    let mut graph = IndexedTxGraph::<ConfirmationBlockTime, KeychainTxOutIndex<String>>::new(
         KeychainTxOutIndex::new(10),
     );
 
-    graph.index.add_keychain("keychain_1".into(), desc_1);
-    graph.index.add_keychain("keychain_2".into(), desc_2);
+    assert!(graph
+        .index
+        .insert_descriptor("keychain_1".into(), desc_1)
+        .unwrap());
+    assert!(graph
+        .index
+        .insert_descriptor("keychain_2".into(), desc_2)
+        .unwrap());
 
     // Get trusted and untrusted addresses
 
@@ -132,16 +155,22 @@ fn test_list_owned_txouts() {
     let mut untrusted_spks: Vec<ScriptBuf> = Vec::new();
 
     {
-        // we need to scope here to take immutanble reference of the graph
+        // we need to scope here to take immutable reference of the graph
         for _ in 0..10 {
-            let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_1".to_string());
+            let ((_, script), _) = graph
+                .index
+                .reveal_next_spk("keychain_1".to_string())
+                .unwrap();
             // TODO Assert indexes
             trusted_spks.push(script.to_owned());
         }
     }
     {
         for _ in 0..10 {
-            let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_2".to_string());
+            let ((_, script), _) = graph
+                .index
+                .reveal_next_spk("keychain_2".to_string())
+                .unwrap();
             untrusted_spks.push(script.to_owned());
         }
     }
@@ -155,7 +184,7 @@ fn test_list_owned_txouts() {
             ..Default::default()
         }],
         output: vec![TxOut {
-            value: 70000,
+            value: Amount::from_sat(70000),
             script_pubkey: trusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
@@ -164,7 +193,7 @@ fn test_list_owned_txouts() {
     // tx2 is an incoming transaction received at untrusted keychain at block 1.
     let tx2 = Transaction {
         output: vec![TxOut {
-            value: 30000,
+            value: Amount::from_sat(30000),
             script_pubkey: untrusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
@@ -173,11 +202,11 @@ fn test_list_owned_txouts() {
     // tx3 spends tx2 and gives a change back in trusted keychain. Confirmed at Block 2.
     let tx3 = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx2.txid(), 0),
+            previous_output: OutPoint::new(tx2.compute_txid(), 0),
             ..Default::default()
         }],
         output: vec![TxOut {
-            value: 10000,
+            value: Amount::from_sat(10000),
             script_pubkey: trusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
@@ -186,16 +215,16 @@ fn test_list_owned_txouts() {
     // tx4 is an external transaction receiving at untrusted keychain, unconfirmed.
     let tx4 = Transaction {
         output: vec![TxOut {
-            value: 20000,
+            value: Amount::from_sat(20000),
             script_pubkey: untrusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
     };
 
-    // tx5 is spending tx3 and receiving change at trusted keychain, unconfirmed.
+    // tx5 is an external transaction receiving at trusted keychain, unconfirmed.
     let tx5 = Transaction {
         output: vec![TxOut {
-            value: 15000,
+            value: Amount::from_sat(15000),
             script_pubkey: trusted_spks[2].to_owned(),
         }],
         ..common::new_tx(0)
@@ -205,7 +234,7 @@ fn test_list_owned_txouts() {
     let tx6 = common::new_tx(0);
 
     // Insert transactions into graph with respective anchors
-    // For unconfirmed txs we pass in `None`.
+    // Insert unconfirmed txs with a last_seen timestamp
 
     let _ =
         graph.batch_insert_relevant([&tx1, &tx2, &tx3, &tx6].iter().enumerate().map(|(i, tx)| {
@@ -213,13 +242,11 @@ fn test_list_owned_txouts() {
             (
                 *tx,
                 local_chain
-                    .blocks()
-                    .get(&height)
-                    .cloned()
-                    .map(|hash| BlockId { height, hash })
-                    .map(|anchor_block| ConfirmationHeightAnchor {
-                        anchor_block,
-                        confirmation_height: anchor_block.height,
+                    .get(height)
+                    .map(|cp| cp.block_id())
+                    .map(|block_id| ConfirmationBlockTime {
+                        block_id,
+                        confirmation_time: 100,
                     }),
             )
         }));
@@ -228,12 +255,10 @@ fn test_list_owned_txouts() {
 
     // A helper lambda to extract and filter data from the graph.
     let fetch =
-        |height: u32,
-         graph: &IndexedTxGraph<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>| {
+        |height: u32, graph: &IndexedTxGraph<ConfirmationBlockTime, KeychainTxOutIndex<String>>| {
             let chain_tip = local_chain
-                .blocks()
-                .get(&height)
-                .map(|&hash| BlockId { height, hash })
+                .get(height)
+                .map(|cp| cp.block_id())
                 .unwrap_or_else(|| panic!("block must exist at {}", height));
             let txouts = graph
                 .graph()
@@ -257,12 +282,9 @@ fn test_list_owned_txouts() {
                 &local_chain,
                 chain_tip,
                 graph.index.outpoints().iter().cloned(),
-                |_, spk: &Script| trusted_spks.contains(&spk.to_owned()),
+                |_, spk: ScriptBuf| trusted_spks.contains(&spk),
             );
 
-            assert_eq!(txouts.len(), 5);
-            assert_eq!(utxos.len(), 4);
-
             let confirmed_txouts_txid = txouts
                 .iter()
                 .filter_map(|(_, full_txout)| {
@@ -328,25 +350,27 @@ fn test_list_owned_txouts() {
             balance,
         ) = fetch(0, &graph);
 
-        assert_eq!(confirmed_txouts_txid, [tx1.txid()].into());
+        // tx1 is a confirmed txout and is unspent
+        // tx4, tx5 are unconfirmed
+        assert_eq!(confirmed_txouts_txid, [tx1.compute_txid()].into());
         assert_eq!(
             unconfirmed_txouts_txid,
-            [tx2.txid(), tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
-        assert_eq!(confirmed_utxos_txid, [tx1.txid()].into());
+        assert_eq!(confirmed_utxos_txid, [tx1.compute_txid()].into());
         assert_eq!(
             unconfirmed_utxos_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 25000,   // tx3 + tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 0              // Nothing is confirmed yet
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::ZERO                     // Nothing is confirmed yet
             }
         );
     }
@@ -362,26 +386,32 @@ fn test_list_owned_txouts() {
         ) = fetch(1, &graph);
 
         // tx2 gets into confirmed txout set
-        assert_eq!(confirmed_txouts_txid, [tx1.txid(), tx2.txid()].into());
+        assert_eq!(
+            confirmed_txouts_txid,
+            [tx1.compute_txid(), tx2.compute_txid()].into()
+        );
         assert_eq!(
             unconfirmed_txouts_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
-        // tx2 doesn't get into confirmed utxos set
-        assert_eq!(confirmed_utxos_txid, [tx1.txid()].into());
+        // tx2 gets into confirmed utxos set
+        assert_eq!(
+            confirmed_utxos_txid,
+            [tx1.compute_txid(), tx2.compute_txid()].into()
+        );
         assert_eq!(
             unconfirmed_utxos_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 25000,   // tx3 + tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 0              // Nothing is confirmed yet
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(30_000)         // tx2 got confirmed
             }
         );
     }
@@ -399,21 +429,30 @@ fn test_list_owned_txouts() {
         // tx3 now gets into the confirmed txout set
         assert_eq!(
             confirmed_txouts_txid,
-            [tx1.txid(), tx2.txid(), tx3.txid()].into()
+            [tx1.compute_txid(), tx2.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_txouts_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
-        assert_eq!(unconfirmed_txouts_txid, [tx4.txid(), tx5.txid()].into());
 
         // tx3 also gets into confirmed utxo set
-        assert_eq!(confirmed_utxos_txid, [tx1.txid(), tx3.txid()].into());
-        assert_eq!(unconfirmed_utxos_txid, [tx4.txid(), tx5.txid()].into());
+        assert_eq!(
+            confirmed_utxos_txid,
+            [tx1.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_utxos_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
+        );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 10000          // tx3 got confirmed
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(10000)          // tx3 got confirmed
             }
         );
     }
@@ -428,40 +467,194 @@ fn test_list_owned_txouts() {
             balance,
         ) = fetch(98, &graph);
 
+        // no change compared to block 2
         assert_eq!(
             confirmed_txouts_txid,
-            [tx1.txid(), tx2.txid(), tx3.txid()].into()
+            [tx1.compute_txid(), tx2.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_txouts_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
-        assert_eq!(unconfirmed_txouts_txid, [tx4.txid(), tx5.txid()].into());
 
-        assert_eq!(confirmed_utxos_txid, [tx1.txid(), tx3.txid()].into());
-        assert_eq!(unconfirmed_utxos_txid, [tx4.txid(), tx5.txid()].into());
+        assert_eq!(
+            confirmed_utxos_txid,
+            [tx1.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_utxos_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
+        );
 
         // Coinbase is still immature
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 10000          // tx1 got matured
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(10000)          // tx3 is confirmed
             }
         );
     }
 
     // AT Block 99
     {
-        let (_, _, _, _, balance) = fetch(100, &graph);
+        let (_, _, _, _, balance) = fetch(99, &graph);
 
         // Coinbase maturity hits
         assert_eq!(
             balance,
             Balance {
-                immature: 0,              // coinbase matured
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 80000          // tx1 + tx3
+                immature: Amount::ZERO,                     // coinbase matured
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(80000)          // tx1 + tx3
             }
         );
     }
 }
+
+/// Given a `LocalChain`, `IndexedTxGraph`, and a `Transaction`, when we insert some anchor
+/// (possibly non-canonical) and/or a last-seen timestamp into the graph, we expect the
+/// result of `get_chain_position` in these cases:
+///
+/// - tx with no anchors or last_seen has no `ChainPosition`
+/// - tx with any last_seen will be `Unconfirmed`
+/// - tx with an anchor in best chain will be `Confirmed`
+/// - tx with an anchor not in best chain (no last_seen) has no `ChainPosition`
+#[test]
+fn test_get_chain_position() {
+    use bdk_chain::local_chain::CheckPoint;
+    use bdk_chain::spk_txout::SpkTxOutIndex;
+    use bdk_chain::BlockId;
+
+    struct TestCase<A> {
+        name: &'static str,
+        tx: Transaction,
+        anchor: Option<A>,
+        last_seen: Option<u64>,
+        exp_pos: Option<ChainPosition<A>>,
+    }
+
+    // addr: bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm
+    let spk = ScriptBuf::from_hex("0014c692ecf13534982a9a2834565cbd37add8027140").unwrap();
+    let mut graph = IndexedTxGraph::new({
+        let mut index = SpkTxOutIndex::default();
+        let _ = index.insert_spk(0u32, spk.clone());
+        index
+    });
+
+    // Anchors to test
+    let blocks = vec![block_id!(0, "g"), block_id!(1, "A"), block_id!(2, "B")];
+
+    let cp = CheckPoint::from_block_ids(blocks.clone()).unwrap();
+    let chain = LocalChain::from_tip(cp).unwrap();
+
+    // The test will insert a transaction into the indexed tx graph
+    // along with any anchors and timestamps, then check the value
+    // returned by `get_chain_position`.
+    fn run(
+        chain: &LocalChain,
+        graph: &mut IndexedTxGraph<BlockId, SpkTxOutIndex<u32>>,
+        test: TestCase<BlockId>,
+    ) {
+        let TestCase {
+            name,
+            tx,
+            anchor,
+            last_seen,
+            exp_pos,
+        } = test;
+
+        // add data to graph
+        let txid = tx.compute_txid();
+        let _ = graph.insert_tx(tx);
+        if let Some(anchor) = anchor {
+            let _ = graph.insert_anchor(txid, anchor);
+        }
+        if let Some(seen_at) = last_seen {
+            let _ = graph.insert_seen_at(txid, seen_at);
+        }
+
+        // check chain position
+        let res = graph
+            .graph()
+            .get_chain_position(chain, chain.tip().block_id(), txid);
+        assert_eq!(
+            res.map(ChainPosition::cloned),
+            exp_pos,
+            "failed test case: {name}"
+        );
+    }
+
+    [
+        TestCase {
+            name: "tx no anchors or last_seen - no chain pos",
+            tx: Transaction {
+                output: vec![TxOut {
+                    value: Amount::ONE_BTC,
+                    script_pubkey: spk.clone(),
+                }],
+                ..common::new_tx(0)
+            },
+            anchor: None,
+            last_seen: None,
+            exp_pos: None,
+        },
+        TestCase {
+            name: "tx last_seen - unconfirmed",
+            tx: Transaction {
+                output: vec![TxOut {
+                    value: Amount::ONE_BTC,
+                    script_pubkey: spk.clone(),
+                }],
+                ..common::new_tx(1)
+            },
+            anchor: None,
+            last_seen: Some(2),
+            exp_pos: Some(ChainPosition::Unconfirmed(2)),
+        },
+        TestCase {
+            name: "tx anchor in best chain - confirmed",
+            tx: Transaction {
+                output: vec![TxOut {
+                    value: Amount::ONE_BTC,
+                    script_pubkey: spk.clone(),
+                }],
+                ..common::new_tx(2)
+            },
+            anchor: Some(blocks[1]),
+            last_seen: None,
+            exp_pos: Some(ChainPosition::Confirmed(blocks[1])),
+        },
+        TestCase {
+            name: "tx unknown anchor with last_seen - unconfirmed",
+            tx: Transaction {
+                output: vec![TxOut {
+                    value: Amount::ONE_BTC,
+                    script_pubkey: spk.clone(),
+                }],
+                ..common::new_tx(3)
+            },
+            anchor: Some(block_id!(2, "B'")),
+            last_seen: Some(2),
+            exp_pos: Some(ChainPosition::Unconfirmed(2)),
+        },
+        TestCase {
+            name: "tx unknown anchor - no chain pos",
+            tx: Transaction {
+                output: vec![TxOut {
+                    value: Amount::ONE_BTC,
+                    script_pubkey: spk.clone(),
+                }],
+                ..common::new_tx(4)
+            },
+            anchor: Some(block_id!(2, "B'")),
+            last_seen: None,
+            exp_pos: None,
+        },
+    ]
+    .into_iter()
+    .for_each(|t| run(&chain, &mut graph, t));
+}

crates/chain/tests/test_keychain_txout_index.rs

@@ -4,37 +4,43 @@
 mod common;
 use bdk_chain::{
     collections::BTreeMap,
-    indexed_tx_graph::Indexer,
-    keychain::{self, KeychainTxOutIndex},
-    Append,
+    indexer::keychain_txout::{ChangeSet, KeychainTxOutIndex},
+    DescriptorExt, DescriptorId, Indexer, Merge,
 };
 
-use bitcoin::{secp256k1::Secp256k1, OutPoint, ScriptBuf, Transaction, TxOut};
+use bitcoin::{secp256k1::Secp256k1, Amount, OutPoint, ScriptBuf, Transaction, TxOut};
 use miniscript::{Descriptor, DescriptorPublicKey};
 
+use crate::common::DESCRIPTORS;
+
 #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd)]
 enum TestKeychain {
     External,
     Internal,
 }
 
-fn init_txout_index(
-    lookahead: u32,
-) -> (
-    bdk_chain::keychain::KeychainTxOutIndex<TestKeychain>,
-    Descriptor<DescriptorPublicKey>,
-    Descriptor<DescriptorPublicKey>,
-) {
-    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::new(lookahead);
-
+fn parse_descriptor(descriptor: &str) -> Descriptor<DescriptorPublicKey> {
     let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
-    let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
-    let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
+    Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, descriptor)
+        .unwrap()
+        .0
+}
 
-    txout_index.add_keychain(TestKeychain::External, external_descriptor.clone());
-    txout_index.add_keychain(TestKeychain::Internal, internal_descriptor.clone());
+fn init_txout_index(
+    external_descriptor: Descriptor<DescriptorPublicKey>,
+    internal_descriptor: Descriptor<DescriptorPublicKey>,
+    lookahead: u32,
+) -> KeychainTxOutIndex<TestKeychain> {
+    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(lookahead);
 
-    (txout_index, external_descriptor, internal_descriptor)
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::External, external_descriptor)
+        .unwrap();
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::Internal, internal_descriptor)
+        .unwrap();
+
+    txout_index
 }
 
 fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> ScriptBuf {
@@ -44,29 +50,88 @@ fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> Scr
         .script_pubkey()
 }
 
+// We create two empty changesets lhs and rhs, we then insert various descriptors with various
+// last_revealed, merge rhs to lhs, and check that the result is consistent with these rules:
+// - Existing index doesn't update if the new index in `other` is lower than `self`.
+// - Existing index updates if the new index in `other` is higher than `self`.
+// - Existing index is unchanged if keychain doesn't exist in `other`.
+// - New keychain gets added if the keychain is in `other` but not in `self`.
+#[test]
+fn merge_changesets_check_last_revealed() {
+    let secp = bitcoin::secp256k1::Secp256k1::signing_only();
+    let descriptor_ids: Vec<_> = DESCRIPTORS
+        .iter()
+        .take(4)
+        .map(|d| {
+            Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, d)
+                .unwrap()
+                .0
+                .descriptor_id()
+        })
+        .collect();
+
+    let mut lhs_di = BTreeMap::<DescriptorId, u32>::default();
+    let mut rhs_di = BTreeMap::<DescriptorId, u32>::default();
+    lhs_di.insert(descriptor_ids[0], 7);
+    lhs_di.insert(descriptor_ids[1], 0);
+    lhs_di.insert(descriptor_ids[2], 3);
+
+    rhs_di.insert(descriptor_ids[0], 3); // value less than lhs desc 0
+    rhs_di.insert(descriptor_ids[1], 5); // value more than lhs desc 1
+    lhs_di.insert(descriptor_ids[3], 4); // key doesn't exist in lhs
+
+    let mut lhs = ChangeSet {
+        last_revealed: lhs_di,
+    };
+    let rhs = ChangeSet {
+        last_revealed: rhs_di,
+    };
+    lhs.merge(rhs);
+
+    // Existing index doesn't update if the new index in `other` is lower than `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[0]), Some(&7));
+    // Existing index updates if the new index in `other` is higher than `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[1]), Some(&5));
+    // Existing index is unchanged if keychain doesn't exist in `other`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[2]), Some(&3));
+    // New keychain gets added if the keychain is in `other` but not in `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[3]), Some(&4));
+}
+
 #[test]
 fn test_set_all_derivation_indices() {
-    use bdk_chain::indexed_tx_graph::Indexer;
-
-    let (mut txout_index, _, _) = init_txout_index(0);
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0);
     let derive_to: BTreeMap<_, _> =
         [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into();
+    let last_revealed: BTreeMap<_, _> = [
+        (external_descriptor.descriptor_id(), 12),
+        (internal_descriptor.descriptor_id(), 24),
+    ]
+    .into();
     assert_eq!(
-        txout_index.reveal_to_target_multi(&derive_to).1.as_inner(),
-        &derive_to
+        txout_index.reveal_to_target_multi(&derive_to),
+        ChangeSet {
+            last_revealed: last_revealed.clone()
+        }
     );
-    assert_eq!(txout_index.last_revealed_indices(), &derive_to);
+    assert_eq!(txout_index.last_revealed_indices(), derive_to);
     assert_eq!(
-        txout_index.reveal_to_target_multi(&derive_to).1,
-        keychain::ChangeSet::default(),
+        txout_index.reveal_to_target_multi(&derive_to),
+        ChangeSet::default(),
         "no changes if we set to the same thing"
     );
-    assert_eq!(txout_index.initial_changeset().as_inner(), &derive_to);
+    assert_eq!(txout_index.initial_changeset().last_revealed, last_revealed);
 }
 
 #[test]
 fn test_lookahead() {
-    let (mut txout_index, external_desc, internal_desc) = init_txout_index(10);
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 10);
 
     // given:
     // - external lookahead set to 10
@@ -76,15 +141,16 @@ fn test_lookahead() {
     // - scripts cached in spk_txout_index should increase correctly
     // - stored scripts of external keychain should be of expected counts
     for index in (0..20).skip_while(|i| i % 2 == 1) {
-        let (revealed_spks, revealed_changeset) =
-            txout_index.reveal_to_target(&TestKeychain::External, index);
+        let (revealed_spks, revealed_changeset) = txout_index
+            .reveal_to_target(TestKeychain::External, index)
+            .unwrap();
         assert_eq!(
-            revealed_spks.collect::<Vec<_>>(),
-            vec![(index, spk_at_index(&external_desc, index))],
+            revealed_spks,
+            vec![(index, spk_at_index(&external_descriptor, index))],
         );
         assert_eq!(
-            revealed_changeset.as_inner(),
-            &[(TestKeychain::External, index)].into()
+            &revealed_changeset.last_revealed,
+            &[(external_descriptor.descriptor_id(), index)].into()
         );
 
         assert_eq!(
@@ -95,25 +161,25 @@ fn test_lookahead() {
         );
         assert_eq!(
             txout_index
-                .revealed_keychain_spks(&TestKeychain::External)
+                .revealed_keychain_spks(TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_keychain_spks(&TestKeychain::Internal)
+                .revealed_keychain_spks(TestKeychain::Internal)
                 .count(),
             0,
         );
         assert_eq!(
             txout_index
-                .unused_keychain_spks(&TestKeychain::External)
+                .unused_keychain_spks(TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .unused_keychain_spks(&TestKeychain::Internal)
+                .unused_keychain_spks(TestKeychain::Internal)
                 .count(),
             0,
         );
@@ -126,17 +192,18 @@ fn test_lookahead() {
     // - derivation index is set ahead of current derivation index + lookahead
     // expect:
     // - scripts cached in spk_txout_index should increase correctly, a.k.a. no scripts are skipped
-    let (revealed_spks, revealed_changeset) =
-        txout_index.reveal_to_target(&TestKeychain::Internal, 24);
+    let (revealed_spks, revealed_changeset) = txout_index
+        .reveal_to_target(TestKeychain::Internal, 24)
+        .unwrap();
     assert_eq!(
-        revealed_spks.collect::<Vec<_>>(),
+        revealed_spks,
         (0..=24)
-            .map(|index| (index, spk_at_index(&internal_desc, index)))
+            .map(|index| (index, spk_at_index(&internal_descriptor, index)))
             .collect::<Vec<_>>(),
     );
     assert_eq!(
-        revealed_changeset.as_inner(),
-        &[(TestKeychain::Internal, 24)].into()
+        &revealed_changeset.last_revealed,
+        &[(internal_descriptor.descriptor_id(), 24)].into()
     );
     assert_eq!(
         txout_index.inner().all_spks().len(),
@@ -147,17 +214,17 @@ fn test_lookahead() {
     );
     assert_eq!(
         txout_index
-            .revealed_keychain_spks(&TestKeychain::Internal)
+            .revealed_keychain_spks(TestKeychain::Internal)
             .count(),
         25,
     );
 
     // ensure derivation indices are expected for each keychain
     let last_external_index = txout_index
-        .last_revealed_index(&TestKeychain::External)
+        .last_revealed_index(TestKeychain::External)
         .expect("already derived");
     let last_internal_index = txout_index
-        .last_revealed_index(&TestKeychain::Internal)
+        .last_revealed_index(TestKeychain::Internal)
         .expect("already derived");
     assert_eq!(last_external_index, 19);
     assert_eq!(last_internal_index, 24);
@@ -172,40 +239,40 @@ fn test_lookahead() {
         let tx = Transaction {
             output: vec![
                 TxOut {
-                    script_pubkey: external_desc
+                    script_pubkey: external_descriptor
                         .at_derivation_index(external_index)
                         .unwrap()
                         .script_pubkey(),
-                    value: 10_000,
+                    value: Amount::from_sat(10_000),
                 },
                 TxOut {
-                    script_pubkey: internal_desc
+                    script_pubkey: internal_descriptor
                         .at_derivation_index(internal_index)
                         .unwrap()
                         .script_pubkey(),
-                    value: 10_000,
+                    value: Amount::from_sat(10_000),
                 },
             ],
             ..common::new_tx(external_index)
         };
-        assert_eq!(txout_index.index_tx(&tx), keychain::ChangeSet::default());
+        assert_eq!(txout_index.index_tx(&tx), ChangeSet::default());
         assert_eq!(
-            txout_index.last_revealed_index(&TestKeychain::External),
+            txout_index.last_revealed_index(TestKeychain::External),
             Some(last_external_index)
         );
         assert_eq!(
-            txout_index.last_revealed_index(&TestKeychain::Internal),
+            txout_index.last_revealed_index(TestKeychain::Internal),
             Some(last_internal_index)
         );
         assert_eq!(
             txout_index
-                .revealed_keychain_spks(&TestKeychain::External)
+                .revealed_keychain_spks(TestKeychain::External)
                 .count(),
             last_external_index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_keychain_spks(&TestKeychain::Internal)
+                .revealed_keychain_spks(TestKeychain::Internal)
                 .count(),
             last_internal_index as usize + 1,
         );
@@ -219,14 +286,17 @@ fn test_lookahead() {
 // - last used index should change as expected
 #[test]
 fn test_scan_with_lookahead() {
-    let (mut txout_index, external_desc, _) = init_txout_index(10);
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 10);
 
     let spks: BTreeMap<u32, ScriptBuf> = [0, 10, 20, 30]
         .into_iter()
         .map(|i| {
             (
                 i,
-                external_desc
+                external_descriptor
                     .at_derivation_index(i)
                     .unwrap()
                     .script_pubkey(),
@@ -238,33 +308,33 @@ fn test_scan_with_lookahead() {
         let op = OutPoint::new(h!("fake tx"), spk_i);
         let txout = TxOut {
             script_pubkey: spk.clone(),
-            value: 0,
+            value: Amount::ZERO,
         };
 
         let changeset = txout_index.index_txout(op, &txout);
         assert_eq!(
-            changeset.as_inner(),
-            &[(TestKeychain::External, spk_i)].into()
+            &changeset.last_revealed,
+            &[(external_descriptor.descriptor_id(), spk_i)].into()
         );
         assert_eq!(
-            txout_index.last_revealed_index(&TestKeychain::External),
+            txout_index.last_revealed_index(TestKeychain::External),
             Some(spk_i)
         );
         assert_eq!(
-            txout_index.last_used_index(&TestKeychain::External),
+            txout_index.last_used_index(TestKeychain::External),
             Some(spk_i)
         );
     }
 
     // now try with index 41 (lookahead surpassed), we expect that the txout to not be indexed
-    let spk_41 = external_desc
+    let spk_41 = external_descriptor
         .at_derivation_index(41)
         .unwrap()
         .script_pubkey();
     let op = OutPoint::new(h!("fake tx"), 41);
     let txout = TxOut {
         script_pubkey: spk_41,
-        value: 0,
+        value: Amount::ZERO,
     };
     let changeset = txout_index.index_txout(op, &txout);
     assert!(changeset.is_empty());
@@ -273,11 +343,13 @@ fn test_scan_with_lookahead() {
 #[test]
 #[rustfmt::skip]
 fn test_wildcard_derivations() {
-    let (mut txout_index, external_desc, _) = init_txout_index(0);
-    let external_spk_0 = external_desc.at_derivation_index(0).unwrap().script_pubkey();
-    let external_spk_16 = external_desc.at_derivation_index(16).unwrap().script_pubkey();
-    let external_spk_26 = external_desc.at_derivation_index(26).unwrap().script_pubkey();
-    let external_spk_27 = external_desc.at_derivation_index(27).unwrap().script_pubkey();
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index = init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0);
+    let external_spk_0 = external_descriptor.at_derivation_index(0).unwrap().script_pubkey();
+    let external_spk_16 = external_descriptor.at_derivation_index(16).unwrap().script_pubkey();
+    let external_spk_26 = external_descriptor.at_derivation_index(26).unwrap().script_pubkey();
+    let external_spk_27 = external_descriptor.at_derivation_index(27).unwrap().script_pubkey();
 
     // - nothing is derived
     // - unused list is also empty
@@ -285,13 +357,13 @@ fn test_wildcard_derivations() {
     // - next_derivation_index() == (0, true)
     // - derive_new() == ((0, <spk>), keychain::ChangeSet)
     // - next_unused() == ((0, <spk>), keychain::ChangeSet:is_empty())
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, external_spk_0.as_script()));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, external_spk_0.as_script()));
-    assert_eq!(changeset.as_inner(), &[].into());
+    assert_eq!(txout_index.next_index(TestKeychain::External).unwrap(), (0, true));
+    let (spk, changeset) = txout_index.reveal_next_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0_u32, external_spk_0.clone()));
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 0)].into());
+    let (spk, changeset) = txout_index.next_unused_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0_u32, external_spk_0.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // - derived till 25
     // - used all spks till 15.
@@ -301,22 +373,22 @@ fn test_wildcard_derivations() {
     // - next_derivation_index() = (26, true)
     // - derive_new() = ((26, <spk>), keychain::ChangeSet)
     // - next_unused() == ((16, <spk>), keychain::ChangeSet::is_empty())
-    let _ = txout_index.reveal_to_target(&TestKeychain::External, 25);
+    let _ = txout_index.reveal_to_target(TestKeychain::External, 25);
 
     (0..=15)
         .chain([17, 20, 23])
         .for_each(|index| assert!(txout_index.mark_used(TestKeychain::External, index)));
 
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (26, true));
+    assert_eq!(txout_index.next_index(TestKeychain::External).unwrap(), (26, true));
 
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (26, external_spk_26.as_script()));
+    let (spk, changeset) = txout_index.reveal_next_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (26, external_spk_26));
 
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 26)].into());
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 26)].into());
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (16, external_spk_16.as_script()));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index.next_unused_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (16, external_spk_16));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // - Use all the derived till 26.
     // - next_unused() = ((27, <spk>), keychain::ChangeSet)
@@ -324,9 +396,9 @@ fn test_wildcard_derivations() {
         txout_index.mark_used(TestKeychain::External, index);
     });
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (27, external_spk_27.as_script()));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 27)].into());
+    let (spk, changeset) = txout_index.next_unused_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (27, external_spk_27));
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 27)].into());
 }
 
 #[test]
@@ -334,13 +406,16 @@ fn test_non_wildcard_derivations() {
     let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
     let secp = bitcoin::secp256k1::Secp256k1::signing_only();
-    let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
+    let (no_wildcard_descriptor, _) =
+        Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, DESCRIPTORS[6]).unwrap();
     let external_spk = no_wildcard_descriptor
         .at_derivation_index(0)
         .unwrap()
         .script_pubkey();
 
-    txout_index.add_keychain(TestKeychain::External, no_wildcard_descriptor);
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::External, no_wildcard_descriptor.clone())
+        .unwrap();
 
     // given:
     // - `txout_index` with no stored scripts
@@ -348,14 +423,20 @@ fn test_non_wildcard_derivations() {
     // - next derivation index should be new
     // - when we derive a new script, script @ index 0
     // - when we get the next unused script, script @ index 0
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, external_spk.as_script()));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
+    assert_eq!(
+        txout_index.next_index(TestKeychain::External).unwrap(),
+        (0, true)
+    );
+    let (spk, changeset) = txout_index.reveal_next_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(
+        &changeset.last_revealed,
+        &[(no_wildcard_descriptor.descriptor_id(), 0)].into()
+    );
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, external_spk.as_script()));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index.next_unused_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // given:
     // - the non-wildcard descriptor already has a stored and used script
@@ -363,26 +444,246 @@ fn test_non_wildcard_derivations() {
     // - next derivation index should not be new
     // - derive new and next unused should return the old script
     // - store_up_to should not panic and return empty changeset
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, false));
+    assert_eq!(
+        txout_index.next_index(TestKeychain::External).unwrap(),
+        (0, false)
+    );
     txout_index.mark_used(TestKeychain::External, 0);
 
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, external_spk.as_script()));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index.reveal_next_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, external_spk.as_script()));
-    assert_eq!(changeset.as_inner(), &[].into());
-    let (revealed_spks, revealed_changeset) =
-        txout_index.reveal_to_target(&TestKeychain::External, 200);
-    assert_eq!(revealed_spks.count(), 0);
+    let (spk, changeset) = txout_index.next_unused_spk(TestKeychain::External).unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
+    let (revealed_spks, revealed_changeset) = txout_index
+        .reveal_to_target(TestKeychain::External, 200)
+        .unwrap();
+    assert_eq!(revealed_spks.len(), 0);
     assert!(revealed_changeset.is_empty());
 
     // we check that spks_of_keychain returns a SpkIterator with just one element
     assert_eq!(
         txout_index
-            .revealed_keychain_spks(&TestKeychain::External)
+            .revealed_keychain_spks(TestKeychain::External)
             .count(),
         1,
     );
 }
+
+/// Check that calling `lookahead_to_target` stores the expected spks.
+#[test]
+fn lookahead_to_target() {
+    #[derive(Default)]
+    struct TestCase {
+        /// Global lookahead value.
+        lookahead: u32,
+        /// Last revealed index for external keychain.
+        external_last_revealed: Option<u32>,
+        /// Last revealed index for internal keychain.
+        internal_last_revealed: Option<u32>,
+        /// Call `lookahead_to_target(External, u32)`.
+        external_target: Option<u32>,
+        /// Call `lookahead_to_target(Internal, u32)`.
+        internal_target: Option<u32>,
+    }
+
+    let test_cases = &[
+        TestCase {
+            lookahead: 0,
+            external_target: Some(100),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 10,
+            internal_target: Some(99),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 100,
+            internal_target: Some(9),
+            external_target: Some(10),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 12,
+            external_last_revealed: Some(2),
+            internal_last_revealed: Some(2),
+            internal_target: Some(15),
+            external_target: Some(13),
+        },
+        TestCase {
+            lookahead: 13,
+            external_last_revealed: Some(100),
+            internal_last_revealed: Some(21),
+            internal_target: Some(120),
+            external_target: Some(130),
+        },
+    ];
+
+    for t in test_cases {
+        let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+        let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+        let mut index = init_txout_index(
+            external_descriptor.clone(),
+            internal_descriptor.clone(),
+            t.lookahead,
+        );
+
+        if let Some(last_revealed) = t.external_last_revealed {
+            let _ = index.reveal_to_target(TestKeychain::External, last_revealed);
+        }
+        if let Some(last_revealed) = t.internal_last_revealed {
+            let _ = index.reveal_to_target(TestKeychain::Internal, last_revealed);
+        }
+
+        let keychain_test_cases = [
+            (
+                TestKeychain::External,
+                t.external_last_revealed,
+                t.external_target,
+            ),
+            (
+                TestKeychain::Internal,
+                t.internal_last_revealed,
+                t.internal_target,
+            ),
+        ];
+        for (keychain, last_revealed, target) in keychain_test_cases {
+            if let Some(target) = target {
+                let original_last_stored_index = match last_revealed {
+                    Some(last_revealed) => Some(last_revealed + t.lookahead),
+                    None => t.lookahead.checked_sub(1),
+                };
+                let exp_last_stored_index = match original_last_stored_index {
+                    Some(original_last_stored_index) => {
+                        Ord::max(target, original_last_stored_index)
+                    }
+                    None => target,
+                };
+                index.lookahead_to_target(keychain.clone(), target);
+                let keys = index
+                    .inner()
+                    .all_spks()
+                    .range((keychain.clone(), 0)..=(keychain.clone(), u32::MAX))
+                    .map(|(k, _)| k.clone())
+                    .collect::<Vec<_>>();
+                let exp_keys = core::iter::repeat(keychain)
+                    .zip(0_u32..=exp_last_stored_index)
+                    .collect::<Vec<_>>();
+                assert_eq!(keys, exp_keys);
+            }
+        }
+    }
+}
+
+#[test]
+fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() {
+    let desc = parse_descriptor(DESCRIPTORS[0]);
+    let changesets: &[ChangeSet] = &[
+        ChangeSet {
+            last_revealed: [(desc.descriptor_id(), 10)].into(),
+        },
+        ChangeSet {
+            last_revealed: [(desc.descriptor_id(), 12)].into(),
+        },
+    ];
+
+    let mut indexer_a = KeychainTxOutIndex::<TestKeychain>::new(0);
+    indexer_a
+        .insert_descriptor(TestKeychain::External, desc.clone())
+        .expect("must insert keychain");
+    for changeset in changesets {
+        indexer_a.apply_changeset(changeset.clone());
+    }
+
+    let mut indexer_b = KeychainTxOutIndex::<TestKeychain>::new(0);
+    indexer_b
+        .insert_descriptor(TestKeychain::External, desc.clone())
+        .expect("must insert keychain");
+    let aggregate_changesets = changesets
+        .iter()
+        .cloned()
+        .reduce(|mut agg, cs| {
+            agg.merge(cs);
+            agg
+        })
+        .expect("must aggregate changesets");
+    indexer_b.apply_changeset(aggregate_changesets);
+
+    assert_eq!(
+        indexer_a.keychains().collect::<Vec<_>>(),
+        indexer_b.keychains().collect::<Vec<_>>()
+    );
+    assert_eq!(
+        indexer_a.spk_at_index(TestKeychain::External, 0),
+        indexer_b.spk_at_index(TestKeychain::External, 0)
+    );
+    assert_eq!(
+        indexer_a.spk_at_index(TestKeychain::Internal, 0),
+        indexer_b.spk_at_index(TestKeychain::Internal, 0)
+    );
+    assert_eq!(
+        indexer_a.last_revealed_indices(),
+        indexer_b.last_revealed_indices()
+    );
+}
+
+#[test]
+fn assigning_same_descriptor_to_multiple_keychains_should_error() {
+    let desc = parse_descriptor(DESCRIPTORS[0]);
+    let mut indexer = KeychainTxOutIndex::<TestKeychain>::new(0);
+    let _ = indexer
+        .insert_descriptor(TestKeychain::Internal, desc.clone())
+        .unwrap();
+    assert!(indexer
+        .insert_descriptor(TestKeychain::External, desc)
+        .is_err())
+}
+
+#[test]
+fn reassigning_keychain_to_a_new_descriptor_should_error() {
+    let desc1 = parse_descriptor(DESCRIPTORS[0]);
+    let desc2 = parse_descriptor(DESCRIPTORS[1]);
+    let mut indexer = KeychainTxOutIndex::<TestKeychain>::new(0);
+    let _ = indexer.insert_descriptor(TestKeychain::Internal, desc1);
+    assert!(indexer
+        .insert_descriptor(TestKeychain::Internal, desc2)
+        .is_err());
+}
+
+#[test]
+fn when_querying_over_a_range_of_keychains_the_utxos_should_show_up() {
+    let mut indexer = KeychainTxOutIndex::<usize>::new(0);
+    let mut tx = common::new_tx(0);
+
+    for (i, descriptor) in DESCRIPTORS.iter().enumerate() {
+        let descriptor = parse_descriptor(descriptor);
+        let _ = indexer.insert_descriptor(i, descriptor.clone()).unwrap();
+        if i != 4 {
+            // skip one in the middle to see if uncovers any bugs
+            indexer.reveal_next_spk(i);
+        }
+        tx.output.push(TxOut {
+            script_pubkey: descriptor.at_derivation_index(0).unwrap().script_pubkey(),
+            value: Amount::from_sat(10_000),
+        });
+    }
+
+    let n_spks = DESCRIPTORS.len() - /*we skipped one*/ 1;
+
+    let _ = indexer.index_tx(&tx);
+    assert_eq!(indexer.outpoints().len(), n_spks);
+
+    assert_eq!(indexer.revealed_spks(0..DESCRIPTORS.len()).count(), n_spks);
+    assert_eq!(indexer.revealed_spks(1..4).count(), 4 - 1);
+    assert_eq!(
+        indexer.net_value(&tx, 0..DESCRIPTORS.len()).to_sat(),
+        (10_000 * n_spks) as i64
+    );
+    assert_eq!(
+        indexer.net_value(&tx, 3..6).to_sat(),
+        (10_000 * (6 - 3 - /*the skipped one*/ 1)) as i64
+    );
+}

crates/chain/tests/test_local_chain.rs

@@ -1,11 +1,16 @@
+#![cfg(feature = "miniscript")]
+
+use std::ops::{Bound, RangeBounds};
+
 use bdk_chain::{
     local_chain::{
         AlterCheckPointError, ApplyHeaderError, CannotConnectError, ChangeSet, CheckPoint,
-        LocalChain, MissingGenesisError, Update,
+        LocalChain, MissingGenesisError,
     },
     BlockId,
 };
 use bitcoin::{block::Header, hashes::Hash, BlockHash};
+use proptest::prelude::*;
 
 #[macro_use]
 mod common;
@@ -14,7 +19,7 @@ mod common;
 struct TestLocalChain<'a> {
     name: &'static str,
     chain: LocalChain,
-    update: Update,
+    update: CheckPoint,
     exp: ExpectedResult<'a>,
 }
 
@@ -528,6 +533,123 @@ fn checkpoint_from_block_ids() {
     }
 }
 
+#[test]
+fn checkpoint_query() {
+    struct TestCase {
+        chain: LocalChain,
+        /// The heights we want to call [`CheckPoint::query`] with, represented as an inclusive
+        /// range.
+        ///
+        /// If a [`CheckPoint`] exists at that height, we expect [`CheckPoint::query`] to return
+        /// it. If not, [`CheckPoint::query`] should return `None`.
+        query_range: (u32, u32),
+    }
+
+    let test_cases = [
+        TestCase {
+            chain: local_chain![(0, h!("_")), (1, h!("A"))],
+            query_range: (0, 2),
+        },
+        TestCase {
+            chain: local_chain![(0, h!("_")), (2, h!("B")), (3, h!("C"))],
+            query_range: (0, 3),
+        },
+    ];
+
+    for t in test_cases.into_iter() {
+        let tip = t.chain.tip();
+        for h in t.query_range.0..=t.query_range.1 {
+            let query_result = tip.get(h);
+
+            // perform an exhausitive search for the checkpoint at height `h`
+            let exp_hash = t
+                .chain
+                .iter_checkpoints()
+                .find(|cp| cp.height() == h)
+                .map(|cp| cp.hash());
+
+            match query_result {
+                Some(cp) => {
+                    assert_eq!(Some(cp.hash()), exp_hash);
+                    assert_eq!(cp.height(), h);
+                }
+                None => assert!(exp_hash.is_none()),
+            }
+        }
+    }
+}
+
+#[test]
+fn checkpoint_insert() {
+    struct TestCase<'a> {
+        /// The name of the test.
+        name: &'a str,
+        /// The original checkpoint chain to call [`CheckPoint::insert`] on.
+        chain: &'a [(u32, BlockHash)],
+        /// The `block_id` to insert.
+        to_insert: (u32, BlockHash),
+        /// The expected final checkpoint chain after calling [`CheckPoint::insert`].
+        exp_final_chain: &'a [(u32, BlockHash)],
+    }
+
+    let test_cases = [
+        TestCase {
+            name: "insert_above_tip",
+            chain: &[(1, h!("a")), (2, h!("b"))],
+            to_insert: (4, h!("d")),
+            exp_final_chain: &[(1, h!("a")), (2, h!("b")), (4, h!("d"))],
+        },
+        TestCase {
+            name: "insert_already_exists_expect_no_change",
+            chain: &[(1, h!("a")), (2, h!("b")), (3, h!("c"))],
+            to_insert: (2, h!("b")),
+            exp_final_chain: &[(1, h!("a")), (2, h!("b")), (3, h!("c"))],
+        },
+        TestCase {
+            name: "insert_in_middle",
+            chain: &[(2, h!("b")), (4, h!("d")), (5, h!("e"))],
+            to_insert: (3, h!("c")),
+            exp_final_chain: &[(2, h!("b")), (3, h!("c")), (4, h!("d")), (5, h!("e"))],
+        },
+        TestCase {
+            name: "replace_one",
+            chain: &[(3, h!("c")), (4, h!("d")), (5, h!("e"))],
+            to_insert: (5, h!("E")),
+            exp_final_chain: &[(3, h!("c")), (4, h!("d")), (5, h!("E"))],
+        },
+        TestCase {
+            name: "insert_conflict_should_evict",
+            chain: &[(3, h!("c")), (4, h!("d")), (5, h!("e")), (6, h!("f"))],
+            to_insert: (4, h!("D")),
+            exp_final_chain: &[(3, h!("c")), (4, h!("D"))],
+        },
+    ];
+
+    fn genesis_block() -> impl Iterator<Item = BlockId> {
+        core::iter::once((0, h!("_"))).map(BlockId::from)
+    }
+
+    for (i, t) in test_cases.into_iter().enumerate() {
+        println!("Running [{}] '{}'", i, t.name);
+
+        let chain = CheckPoint::from_block_ids(
+            genesis_block().chain(t.chain.iter().copied().map(BlockId::from)),
+        )
+        .expect("test formed incorrectly, must construct checkpoint chain");
+
+        let exp_final_chain = CheckPoint::from_block_ids(
+            genesis_block().chain(t.exp_final_chain.iter().copied().map(BlockId::from)),
+        )
+        .expect("test formed incorrectly, must construct checkpoint chain");
+
+        assert_eq!(
+            chain.insert(t.to_insert.into()),
+            exp_final_chain,
+            "unexpected final chain"
+        );
+    }
+}
+
 #[test]
 fn local_chain_apply_header_connected_to() {
     fn header_from_prev_blockhash(prev_blockhash: BlockHash) -> Header {
@@ -552,9 +674,9 @@ fn local_chain_apply_header_connected_to() {
 
     let test_cases = [
         {
-            let header = header_from_prev_blockhash(h!("A"));
+            let header = header_from_prev_blockhash(h!("_"));
             let hash = header.block_hash();
-            let height = 2;
+            let height = 1;
             let connected_to = BlockId { height, hash };
             TestCase {
                 name: "connected_to_self_header_applied_to_self",
@@ -679,3 +801,48 @@ fn local_chain_apply_header_connected_to() {
         assert_eq!(result, exp_result, "[{}:{}] unexpected result", i, t.name);
     }
 }
+
+fn generate_height_range_bounds(
+    height_upper_bound: u32,
+) -> impl Strategy<Value = (Bound<u32>, Bound<u32>)> {
+    fn generate_height_bound(height_upper_bound: u32) -> impl Strategy<Value = Bound<u32>> {
+        prop_oneof![
+            (0..height_upper_bound).prop_map(Bound::Included),
+            (0..height_upper_bound).prop_map(Bound::Excluded),
+            Just(Bound::Unbounded),
+        ]
+    }
+    (
+        generate_height_bound(height_upper_bound),
+        generate_height_bound(height_upper_bound),
+    )
+}
+
+fn generate_checkpoints(max_height: u32, max_count: usize) -> impl Strategy<Value = CheckPoint> {
+    proptest::collection::btree_set(1..max_height, 0..max_count).prop_map(|mut heights| {
+        heights.insert(0); // must have genesis
+        CheckPoint::from_block_ids(heights.into_iter().map(|height| {
+            let hash = bitcoin::hashes::Hash::hash(height.to_le_bytes().as_slice());
+            BlockId { height, hash }
+        }))
+        .expect("blocks must be in order as it comes from btreeset")
+    })
+}
+
+proptest! {
+    #![proptest_config(ProptestConfig {
+        ..Default::default()
+    })]
+
+    /// Ensure that [`CheckPoint::range`] returns the expected checkpoint heights by comparing it
+    /// against a more primitive approach.
+    #[test]
+    fn checkpoint_range(
+        range in generate_height_range_bounds(21_000),
+        cp in generate_checkpoints(21_000, 2100)
+    ) {
+        let exp_heights = cp.iter().map(|cp| cp.height()).filter(|h| range.contains(h)).collect::<Vec<u32>>();
+        let heights = cp.range(range).map(|cp| cp.height()).collect::<Vec<u32>>();
+        prop_assert_eq!(heights, exp_heights);
+    }
+}

crates/chain/tests/test_spk_txout_index.rs

@@ -1,5 +1,7 @@
-use bdk_chain::{indexed_tx_graph::Indexer, SpkTxOutIndex};
-use bitcoin::{absolute, OutPoint, ScriptBuf, Transaction, TxIn, TxOut};
+use bdk_chain::{spk_txout::SpkTxOutIndex, Indexer};
+use bitcoin::{
+    absolute, transaction, Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxIn, TxOut,
+};
 
 #[test]
 fn spk_txout_sent_and_received() {
@@ -11,48 +13,70 @@ fn spk_txout_sent_and_received() {
     index.insert_spk(1, spk2.clone());
 
     let tx1 = Transaction {
-        version: 0x02,
+        version: transaction::Version::TWO,
         lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
-            value: 42_000,
+            value: Amount::from_sat(42_000),
             script_pubkey: spk1.clone(),
         }],
     };
 
-    assert_eq!(index.sent_and_received(&tx1), (0, 42_000));
-    assert_eq!(index.net_value(&tx1), 42_000);
+    assert_eq!(
+        index.sent_and_received(&tx1, ..),
+        (Amount::from_sat(0), Amount::from_sat(42_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx1, ..1),
+        (Amount::from_sat(0), Amount::from_sat(42_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx1, 1..),
+        (Amount::from_sat(0), Amount::from_sat(0))
+    );
+    assert_eq!(index.net_value(&tx1, ..), SignedAmount::from_sat(42_000));
     index.index_tx(&tx1);
     assert_eq!(
-        index.sent_and_received(&tx1),
-        (0, 42_000),
+        index.sent_and_received(&tx1, ..),
+        (Amount::from_sat(0), Amount::from_sat(42_000)),
         "shouldn't change after scanning"
     );
 
     let tx2 = Transaction {
-        version: 0x1,
+        version: transaction::Version::ONE,
         lock_time: absolute::LockTime::ZERO,
         input: vec![TxIn {
             previous_output: OutPoint {
-                txid: tx1.txid(),
+                txid: tx1.compute_txid(),
                 vout: 0,
             },
             ..Default::default()
         }],
         output: vec![
             TxOut {
-                value: 20_000,
+                value: Amount::from_sat(20_000),
                 script_pubkey: spk2,
             },
             TxOut {
                 script_pubkey: spk1,
-                value: 30_000,
+                value: Amount::from_sat(30_000),
             },
         ],
     };
 
-    assert_eq!(index.sent_and_received(&tx2), (42_000, 50_000));
-    assert_eq!(index.net_value(&tx2), 8_000);
+    assert_eq!(
+        index.sent_and_received(&tx2, ..),
+        (Amount::from_sat(42_000), Amount::from_sat(50_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx2, ..1),
+        (Amount::from_sat(42_000), Amount::from_sat(30_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx2, 1..),
+        (Amount::from_sat(0), Amount::from_sat(20_000))
+    );
+    assert_eq!(index.net_value(&tx2, ..), SignedAmount::from_sat(8_000));
 }
 
 #[test]
@@ -73,11 +97,11 @@ fn mark_used() {
     assert!(spk_index.is_used(&1));
 
     let tx1 = Transaction {
-        version: 0x02,
+        version: transaction::Version::TWO,
         lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
-            value: 42_000,
+            value: Amount::from_sat(42_000),
             script_pubkey: spk1,
         }],
     };

crates/chain/tests/test_tx_graph_conflicts.rs

@@ -1,10 +1,12 @@
+#![cfg(feature = "miniscript")]
+
 #[macro_use]
 mod common;
 
 use std::collections::{BTreeSet, HashSet};
 
-use bdk_chain::{keychain::Balance, BlockId};
-use bitcoin::{OutPoint, Script};
+use bdk_chain::{Balance, BlockId};
+use bitcoin::{Amount, OutPoint, ScriptBuf};
 use common::*;
 
 #[allow(dead_code)]
@@ -13,7 +15,7 @@ struct Scenario<'a> {
     name: &'a str,
     /// Transaction templates
     tx_templates: &'a [TxTemplate<'a, BlockId>],
-    /// Names of txs that must exist in the output of `list_chain_txs`
+    /// Names of txs that must exist in the output of `list_canonical_txs`
     exp_chain_txs: HashSet<&'a str>,
     /// Outpoints that must exist in the output of `filter_chain_txouts`
     exp_chain_txouts: HashSet<(&'a str, u32)>,
@@ -25,7 +27,7 @@ struct Scenario<'a> {
 
 /// This test ensures that [`TxGraph`] will reliably filter out irrelevant transactions when
 /// presented with multiple conflicting transaction scenarios using the [`TxTemplate`] structure.
-/// This test also checks that [`TxGraph::list_chain_txs`], [`TxGraph::filter_chain_txouts`],
+/// This test also checks that [`TxGraph::list_canonical_txs`], [`TxGraph::filter_chain_txouts`],
 /// [`TxGraph::filter_chain_unspents`], and [`TxGraph::balance`] return correct data.
 #[test]
 fn test_tx_conflict_handling() {
@@ -79,10 +81,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("confirmed_genesis", 0), ("confirmed_conflict", 0)]),
             exp_unspents: HashSet::from([("confirmed_conflict", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 0,
-                untrusted_pending: 0,
-                confirmed: 20000,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(20000),
             },
         },
         Scenario {
@@ -115,10 +117,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_2", 0)]),
             exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -150,10 +152,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx1", 1), ("tx_conflict_2", 0)]),
             exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -192,10 +194,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_3", 0)]),
             exp_unspents: HashSet::from([("tx_conflict_3", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 40000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(40000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -227,10 +229,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_orphaned_conflict", 0)]),
             exp_unspents: HashSet::from([("tx_orphaned_conflict", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -262,10 +264,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_1", 0)]),
             exp_unspents: HashSet::from([("tx_conflict_1", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 20000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(20000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -311,10 +313,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_confirmed_conflict", 0)]),
             exp_unspents: HashSet::from([("tx_confirmed_conflict", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 0,
-                untrusted_pending: 0,
-                confirmed: 50000,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
             },
         },
         Scenario {
@@ -356,10 +358,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("A", 0), ("B", 0), ("C", 0)]),
             exp_unspents: HashSet::from([("C", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -397,10 +399,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
             exp_unspents: HashSet::from([("B'", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 0,
-                untrusted_pending: 0,
-                confirmed: 20000,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(20000),
             },
         },
         Scenario {
@@ -442,10 +444,10 @@ fn test_tx_conflict_handling() {
             ]),
             exp_unspents: HashSet::from([("C", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -487,10 +489,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
             exp_unspents: HashSet::from([("B'", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 30000,
-                untrusted_pending: 0,
-                confirmed: 0,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
             },
         },
         Scenario {
@@ -532,10 +534,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
             exp_unspents: HashSet::from([("B'", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 0,
-                untrusted_pending: 0,
-                confirmed: 50000,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
             },
         },
         Scenario {
@@ -583,10 +585,10 @@ fn test_tx_conflict_handling() {
             exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
             exp_unspents: HashSet::from([("B'", 0)]),
             exp_balance: Balance {
-                immature: 0,
-                trusted_pending: 0,
-                untrusted_pending: 0,
-                confirmed: 50000,
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
             },
         },
     ];
@@ -595,7 +597,7 @@ fn test_tx_conflict_handling() {
         let (tx_graph, spk_index, exp_tx_ids) = init_graph(scenario.tx_templates.iter());
 
         let txs = tx_graph
-            .list_chain_txs(&local_chain, chain_tip)
+            .list_canonical_txs(&local_chain, chain_tip)
             .map(|tx| tx.tx_node.txid)
             .collect::<BTreeSet<_>>();
         let exp_txs = scenario
@@ -605,7 +607,7 @@ fn test_tx_conflict_handling() {
             .collect::<BTreeSet<_>>();
         assert_eq!(
             txs, exp_txs,
-            "\n[{}] 'list_chain_txs' failed",
+            "\n[{}] 'list_canonical_txs' failed",
             scenario.name
         );
 
@@ -657,7 +659,7 @@ fn test_tx_conflict_handling() {
             &local_chain,
             chain_tip,
             spk_index.outpoints().iter().cloned(),
-            |_, spk: &Script| spk_index.index_of_spk(spk).is_some(),
+            |_, spk: ScriptBuf| spk_index.index_of_spk(spk).is_some(),
         );
         assert_eq!(
             balance, scenario.exp_balance,

crates/electrum/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_electrum"
-version = "0.8.0"
+version = "0.16.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,6 +12,9 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.10.0", default-features = false }
-electrum-client = { version = "0.18" }
+bdk_chain = { path = "../chain", version = "0.17.0" }
+electrum-client = { version = "0.20" }
 #rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] }
+
+[dev-dependencies]
+bdk_testenv = { path = "../testenv", default-features = false }

crates/electrum/src/bdk_electrum_client.rs

@@ -0,0 +1,491 @@
+use bdk_chain::{
+    bitcoin::{block::Header, BlockHash, OutPoint, ScriptBuf, Transaction, Txid},
+    collections::{BTreeMap, HashMap},
+    local_chain::CheckPoint,
+    spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult},
+    tx_graph::TxGraph,
+    Anchor, BlockId, ConfirmationBlockTime,
+};
+use electrum_client::{ElectrumApi, Error, HeaderNotification};
+use std::{
+    collections::BTreeSet,
+    sync::{Arc, Mutex},
+};
+
+/// We include a chain suffix of a certain length for the purpose of robustness.
+const CHAIN_SUFFIX_LENGTH: u32 = 8;
+
+/// Wrapper around an [`electrum_client::ElectrumApi`] which includes an internal in-memory
+/// transaction cache to avoid re-fetching already downloaded transactions.
+#[derive(Debug)]
+pub struct BdkElectrumClient<E> {
+    /// The internal [`electrum_client::ElectrumApi`]
+    pub inner: E,
+    /// The transaction cache
+    tx_cache: Mutex<HashMap<Txid, Arc<Transaction>>>,
+    /// The header cache
+    block_header_cache: Mutex<HashMap<u32, Header>>,
+}
+
+impl<E: ElectrumApi> BdkElectrumClient<E> {
+    /// Creates a new bdk client from a [`electrum_client::ElectrumApi`]
+    pub fn new(client: E) -> Self {
+        Self {
+            inner: client,
+            tx_cache: Default::default(),
+            block_header_cache: Default::default(),
+        }
+    }
+
+    /// Inserts transactions into the transaction cache so that the client will not fetch these
+    /// transactions.
+    pub fn populate_tx_cache<A>(&self, tx_graph: impl AsRef<TxGraph<A>>) {
+        let txs = tx_graph
+            .as_ref()
+            .full_txs()
+            .map(|tx_node| (tx_node.txid, tx_node.tx));
+
+        let mut tx_cache = self.tx_cache.lock().unwrap();
+        for (txid, tx) in txs {
+            tx_cache.insert(txid, tx);
+        }
+    }
+
+    /// Fetch transaction of given `txid`.
+    ///
+    /// If it hits the cache it will return the cached version and avoid making the request.
+    pub fn fetch_tx(&self, txid: Txid) -> Result<Arc<Transaction>, Error> {
+        let tx_cache = self.tx_cache.lock().unwrap();
+
+        if let Some(tx) = tx_cache.get(&txid) {
+            return Ok(Arc::clone(tx));
+        }
+
+        drop(tx_cache);
+
+        let tx = Arc::new(self.inner.transaction_get(&txid)?);
+
+        self.tx_cache.lock().unwrap().insert(txid, Arc::clone(&tx));
+
+        Ok(tx)
+    }
+
+    /// Fetch block header of given `height`.
+    ///
+    /// If it hits the cache it will return the cached version and avoid making the request.
+    fn fetch_header(&self, height: u32) -> Result<Header, Error> {
+        let block_header_cache = self.block_header_cache.lock().unwrap();
+
+        if let Some(header) = block_header_cache.get(&height) {
+            return Ok(*header);
+        }
+
+        drop(block_header_cache);
+
+        self.update_header(height)
+    }
+
+    /// Update a block header at given `height`. Returns the updated header.
+    fn update_header(&self, height: u32) -> Result<Header, Error> {
+        let header = self.inner.block_header(height as usize)?;
+
+        self.block_header_cache
+            .lock()
+            .unwrap()
+            .insert(height, header);
+
+        Ok(header)
+    }
+
+    /// Broadcasts a transaction to the network.
+    ///
+    /// This is a re-export of [`ElectrumApi::transaction_broadcast`].
+    pub fn transaction_broadcast(&self, tx: &Transaction) -> Result<Txid, Error> {
+        self.inner.transaction_broadcast(tx)
+    }
+
+    /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and
+    /// returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `request`: struct with data required to perform a spk-based blockchain client full scan,
+    ///              see [`FullScanRequest`]
+    /// - `stop_gap`: the full scan for each keychain stops after a gap of script pubkeys with no
+    ///              associated transactions
+    /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch
+    ///              request
+    /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee
+    pub fn full_scan<K: Ord + Clone>(
+        &self,
+        request: FullScanRequest<K>,
+        stop_gap: usize,
+        batch_size: usize,
+        fetch_prev_txouts: bool,
+    ) -> Result<FullScanResult<K>, Error> {
+        let (tip, latest_blocks) =
+            fetch_tip_and_latest_blocks(&self.inner, request.chain_tip.clone())?;
+        let mut graph_update = TxGraph::<ConfirmationBlockTime>::default();
+        let mut last_active_indices = BTreeMap::<K, u32>::new();
+
+        for (keychain, spks) in request.spks_by_keychain {
+            if let Some(last_active_index) =
+                self.populate_with_spks(&mut graph_update, spks, stop_gap, batch_size)?
+            {
+                last_active_indices.insert(keychain, last_active_index);
+            }
+        }
+
+        let chain_update = chain_update(tip, &latest_blocks, graph_update.all_anchors())?;
+
+        // Fetch previous `TxOut`s for fee calculation if flag is enabled.
+        if fetch_prev_txouts {
+            self.fetch_prev_txout(&mut graph_update)?;
+        }
+
+        Ok(FullScanResult {
+            graph_update,
+            chain_update,
+            last_active_indices,
+        })
+    }
+
+    /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified
+    /// and returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `request`: struct with data required to perform a spk-based blockchain client sync,
+    ///              see [`SyncRequest`]
+    /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch
+    ///              request
+    /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee
+    ///              calculation
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: Self::full_scan
+    pub fn sync(
+        &self,
+        request: SyncRequest,
+        batch_size: usize,
+        fetch_prev_txouts: bool,
+    ) -> Result<SyncResult, Error> {
+        let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone())
+            .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk)));
+        let mut full_scan_res = self.full_scan(full_scan_req, usize::MAX, batch_size, false)?;
+        let (tip, latest_blocks) =
+            fetch_tip_and_latest_blocks(&self.inner, request.chain_tip.clone())?;
+
+        self.populate_with_txids(&mut full_scan_res.graph_update, request.txids)?;
+        self.populate_with_outpoints(&mut full_scan_res.graph_update, request.outpoints)?;
+
+        let chain_update = chain_update(
+            tip,
+            &latest_blocks,
+            full_scan_res.graph_update.all_anchors(),
+        )?;
+
+        // Fetch previous `TxOut`s for fee calculation if flag is enabled.
+        if fetch_prev_txouts {
+            self.fetch_prev_txout(&mut full_scan_res.graph_update)?;
+        }
+
+        Ok(SyncResult {
+            chain_update,
+            graph_update: full_scan_res.graph_update,
+        })
+    }
+
+    /// Populate the `graph_update` with transactions/anchors associated with the given `spks`.
+    ///
+    /// Transactions that contains an output with requested spk, or spends form an output with
+    /// requested spk will be added to `graph_update`. Anchors of the aforementioned transactions are
+    /// also included.
+    fn populate_with_spks(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationBlockTime>,
+        mut spks: impl Iterator<Item = (u32, ScriptBuf)>,
+        stop_gap: usize,
+        batch_size: usize,
+    ) -> Result<Option<u32>, Error> {
+        let mut unused_spk_count = 0_usize;
+        let mut last_active_index = Option::<u32>::None;
+
+        loop {
+            let spks = (0..batch_size)
+                .map_while(|_| spks.next())
+                .collect::<Vec<_>>();
+            if spks.is_empty() {
+                return Ok(last_active_index);
+            }
+
+            let spk_histories = self
+                .inner
+                .batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?;
+
+            for ((spk_index, _spk), spk_history) in spks.into_iter().zip(spk_histories) {
+                if spk_history.is_empty() {
+                    unused_spk_count = unused_spk_count.saturating_add(1);
+                    if unused_spk_count >= stop_gap {
+                        return Ok(last_active_index);
+                    }
+                    continue;
+                } else {
+                    last_active_index = Some(spk_index);
+                    unused_spk_count = 0;
+                }
+
+                for tx_res in spk_history {
+                    let _ = graph_update.insert_tx(self.fetch_tx(tx_res.tx_hash)?);
+                    self.validate_merkle_for_anchor(graph_update, tx_res.tx_hash, tx_res.height)?;
+                }
+            }
+        }
+    }
+
+    /// Populate the `graph_update` with associated transactions/anchors of `outpoints`.
+    ///
+    /// Transactions in which the outpoint resides, and transactions that spend from the outpoint are
+    /// included. Anchors of the aforementioned transactions are included.
+    fn populate_with_outpoints(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationBlockTime>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+    ) -> Result<(), Error> {
+        for outpoint in outpoints {
+            let op_txid = outpoint.txid;
+            let op_tx = self.fetch_tx(op_txid)?;
+            let op_txout = match op_tx.output.get(outpoint.vout as usize) {
+                Some(txout) => txout,
+                None => continue,
+            };
+            debug_assert_eq!(op_tx.compute_txid(), op_txid);
+
+            // attempt to find the following transactions (alongside their chain positions), and
+            // add to our sparsechain `update`:
+            let mut has_residing = false; // tx in which the outpoint resides
+            let mut has_spending = false; // tx that spends the outpoint
+            for res in self.inner.script_get_history(&op_txout.script_pubkey)? {
+                if has_residing && has_spending {
+                    break;
+                }
+
+                if !has_residing && res.tx_hash == op_txid {
+                    has_residing = true;
+                    let _ = graph_update.insert_tx(Arc::clone(&op_tx));
+                    self.validate_merkle_for_anchor(graph_update, res.tx_hash, res.height)?;
+                }
+
+                if !has_spending && res.tx_hash != op_txid {
+                    let res_tx = self.fetch_tx(res.tx_hash)?;
+                    // we exclude txs/anchors that do not spend our specified outpoint(s)
+                    has_spending = res_tx
+                        .input
+                        .iter()
+                        .any(|txin| txin.previous_output == outpoint);
+                    if !has_spending {
+                        continue;
+                    }
+                    let _ = graph_update.insert_tx(Arc::clone(&res_tx));
+                    self.validate_merkle_for_anchor(graph_update, res.tx_hash, res.height)?;
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Populate the `graph_update` with transactions/anchors of the provided `txids`.
+    fn populate_with_txids(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationBlockTime>,
+        txids: impl IntoIterator<Item = Txid>,
+    ) -> Result<(), Error> {
+        for txid in txids {
+            let tx = match self.fetch_tx(txid) {
+                Ok(tx) => tx,
+                Err(electrum_client::Error::Protocol(_)) => continue,
+                Err(other_err) => return Err(other_err),
+            };
+
+            let spk = tx
+                .output
+                .first()
+                .map(|txo| &txo.script_pubkey)
+                .expect("tx must have an output");
+
+            // because of restrictions of the Electrum API, we have to use the `script_get_history`
+            // call to get confirmation status of our transaction
+            if let Some(r) = self
+                .inner
+                .script_get_history(spk)?
+                .into_iter()
+                .find(|r| r.tx_hash == txid)
+            {
+                self.validate_merkle_for_anchor(graph_update, txid, r.height)?;
+            }
+
+            let _ = graph_update.insert_tx(tx);
+        }
+        Ok(())
+    }
+
+    // Helper function which checks if a transaction is confirmed by validating the merkle proof.
+    // An anchor is inserted if the transaction is validated to be in a confirmed block.
+    fn validate_merkle_for_anchor(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationBlockTime>,
+        txid: Txid,
+        confirmation_height: i32,
+    ) -> Result<(), Error> {
+        if let Ok(merkle_res) = self
+            .inner
+            .transaction_get_merkle(&txid, confirmation_height as usize)
+        {
+            let mut header = self.fetch_header(merkle_res.block_height as u32)?;
+            let mut is_confirmed_tx = electrum_client::utils::validate_merkle_proof(
+                &txid,
+                &header.merkle_root,
+                &merkle_res,
+            );
+
+            // Merkle validation will fail if the header in `block_header_cache` is outdated, so we
+            // want to check if there is a new header and validate against the new one.
+            if !is_confirmed_tx {
+                header = self.update_header(merkle_res.block_height as u32)?;
+                is_confirmed_tx = electrum_client::utils::validate_merkle_proof(
+                    &txid,
+                    &header.merkle_root,
+                    &merkle_res,
+                );
+            }
+
+            if is_confirmed_tx {
+                let _ = graph_update.insert_anchor(
+                    txid,
+                    ConfirmationBlockTime {
+                        confirmation_time: header.time as u64,
+                        block_id: BlockId {
+                            height: merkle_res.block_height as u32,
+                            hash: header.block_hash(),
+                        },
+                    },
+                );
+            }
+        }
+        Ok(())
+    }
+
+    // Helper function which fetches the `TxOut`s of our relevant transactions' previous transactions,
+    // which we do not have by default. This data is needed to calculate the transaction fee.
+    fn fetch_prev_txout(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationBlockTime>,
+    ) -> Result<(), Error> {
+        let full_txs: Vec<Arc<Transaction>> =
+            graph_update.full_txs().map(|tx_node| tx_node.tx).collect();
+        for tx in full_txs {
+            for vin in &tx.input {
+                let outpoint = vin.previous_output;
+                let vout = outpoint.vout;
+                let prev_tx = self.fetch_tx(outpoint.txid)?;
+                let txout = prev_tx.output[vout as usize].clone();
+                let _ = graph_update.insert_txout(outpoint, txout);
+            }
+        }
+        Ok(())
+    }
+}
+
+/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`. The latest blocks are
+/// fetched to construct checkpoint updates with the proper [`BlockHash`] in case of re-org.
+fn fetch_tip_and_latest_blocks(
+    client: &impl ElectrumApi,
+    prev_tip: CheckPoint,
+) -> Result<(CheckPoint, BTreeMap<u32, BlockHash>), Error> {
+    let HeaderNotification { height, .. } = client.block_headers_subscribe()?;
+    let new_tip_height = height as u32;
+
+    // If electrum returns a tip height that is lower than our previous tip, then checkpoints do
+    // not need updating. We just return the previous tip and use that as the point of agreement.
+    if new_tip_height < prev_tip.height() {
+        return Ok((prev_tip, BTreeMap::new()));
+    }
+
+    // Atomically fetch the latest `CHAIN_SUFFIX_LENGTH` count of blocks from Electrum. We use this
+    // to construct our checkpoint update.
+    let mut new_blocks = {
+        let start_height = new_tip_height.saturating_sub(CHAIN_SUFFIX_LENGTH - 1);
+        let hashes = client
+            .block_headers(start_height as _, CHAIN_SUFFIX_LENGTH as _)?
+            .headers
+            .into_iter()
+            .map(|h| h.block_hash());
+        (start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()
+    };
+
+    // Find the "point of agreement" (if any).
+    let agreement_cp = {
+        let mut agreement_cp = Option::<CheckPoint>::None;
+        for cp in prev_tip.iter() {
+            let cp_block = cp.block_id();
+            let hash = match new_blocks.get(&cp_block.height) {
+                Some(&hash) => hash,
+                None => {
+                    assert!(
+                        new_tip_height >= cp_block.height,
+                        "already checked that electrum's tip cannot be smaller"
+                    );
+                    let hash = client.block_header(cp_block.height as _)?.block_hash();
+                    new_blocks.insert(cp_block.height, hash);
+                    hash
+                }
+            };
+            if hash == cp_block.hash {
+                agreement_cp = Some(cp);
+                break;
+            }
+        }
+        agreement_cp
+    };
+
+    let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);
+
+    let new_tip = new_blocks
+        .iter()
+        // Prune `new_blocks` to only include blocks that are actually new.
+        .filter(|(height, _)| Some(*<&u32>::clone(height)) > agreement_height)
+        .map(|(height, hash)| BlockId {
+            height: *height,
+            hash: *hash,
+        })
+        .fold(agreement_cp, |prev_cp, block| {
+            Some(match prev_cp {
+                Some(cp) => cp.push(block).expect("must extend checkpoint"),
+                None => CheckPoint::new(block),
+            })
+        })
+        .expect("must have at least one checkpoint");
+
+    Ok((new_tip, new_blocks))
+}
+
+// Add a corresponding checkpoint per anchor height if it does not yet exist. Checkpoints should not
+// surpass `latest_blocks`.
+fn chain_update<A: Anchor>(
+    mut tip: CheckPoint,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    anchors: &BTreeSet<(A, Txid)>,
+) -> Result<CheckPoint, Error> {
+    for anchor in anchors {
+        let height = anchor.0.anchor_block().height;
+
+        // Checkpoint uses the `BlockHash` from `latest_blocks` so that the hash will be consistent
+        // in case of a re-org.
+        if tip.get(height).is_none() && height <= tip.height() {
+            let hash = match latest_blocks.get(&height) {
+                Some(&hash) => hash,
+                None => anchor.0.anchor_block().hash,
+            };
+            tip = tip.insert(BlockId { hash, height });
+        }
+    }
+    Ok(tip)
+}

crates/electrum/src/electrum_ext.rs

@@ -1,559 +0,0 @@
-use bdk_chain::{
-    bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
-    local_chain::{self, CheckPoint},
-    tx_graph::{self, TxGraph},
-    Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor,
-};
-use electrum_client::{Client, ElectrumApi, Error, HeaderNotification};
-use std::{
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
-    fmt::Debug,
-    str::FromStr,
-};
-
-/// We include a chain suffix of a certain length for the purpose of robustness.
-const CHAIN_SUFFIX_LENGTH: u32 = 8;
-
-/// Represents updates fetched from an Electrum server, but excludes full transactions.
-///
-/// To provide a complete update to [`TxGraph`], you'll need to call [`Self::missing_full_txs`] to
-/// determine the full transactions missing from [`TxGraph`]. Then call [`Self::into_tx_graph`] to
-/// fetch the full transactions from Electrum and finalize the update.
-#[derive(Debug, Default, Clone)]
-pub struct RelevantTxids(HashMap<Txid, BTreeSet<ConfirmationHeightAnchor>>);
-
-impl RelevantTxids {
-    /// Determine the full transactions that are missing from `graph`.
-    ///
-    /// Refer to [`RelevantTxids`] for more details.
-    pub fn missing_full_txs<A: Anchor>(&self, graph: &TxGraph<A>) -> Vec<Txid> {
-        self.0
-            .keys()
-            .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
-            .cloned()
-            .collect()
-    }
-
-    /// Finalizes the [`TxGraph`] update by fetching `missing` txids from the `client`.
-    ///
-    /// Refer to [`RelevantTxids`] for more details.
-    pub fn into_tx_graph(
-        self,
-        client: &Client,
-        seen_at: Option<u64>,
-        missing: Vec<Txid>,
-    ) -> Result<TxGraph<ConfirmationHeightAnchor>, Error> {
-        let new_txs = client.batch_transaction_get(&missing)?;
-        let mut graph = TxGraph::<ConfirmationHeightAnchor>::new(new_txs);
-        for (txid, anchors) in self.0 {
-            if let Some(seen_at) = seen_at {
-                let _ = graph.insert_seen_at(txid, seen_at);
-            }
-            for anchor in anchors {
-                let _ = graph.insert_anchor(txid, anchor);
-            }
-        }
-        Ok(graph)
-    }
-
-    /// Finalizes the update by fetching `missing` txids from the `client`, where the
-    /// resulting [`TxGraph`] has anchors of type [`ConfirmationTimeHeightAnchor`].
-    ///
-    /// Refer to [`RelevantTxids`] for more details.
-    ///
-    /// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
-    // Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
-    // use it.
-    pub fn into_confirmation_time_tx_graph(
-        self,
-        client: &Client,
-        seen_at: Option<u64>,
-        missing: Vec<Txid>,
-    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
-        let graph = self.into_tx_graph(client, seen_at, missing)?;
-
-        let relevant_heights = {
-            let mut visited_heights = HashSet::new();
-            graph
-                .all_anchors()
-                .iter()
-                .map(|(a, _)| a.confirmation_height_upper_bound())
-                .filter(move |&h| visited_heights.insert(h))
-                .collect::<Vec<_>>()
-        };
-
-        let height_to_time = relevant_heights
-            .clone()
-            .into_iter()
-            .zip(
-                client
-                    .batch_block_header(relevant_heights)?
-                    .into_iter()
-                    .map(|bh| bh.time as u64),
-            )
-            .collect::<HashMap<u32, u64>>();
-
-        let graph_changeset = {
-            let old_changeset = TxGraph::default().apply_update(graph);
-            tx_graph::ChangeSet {
-                txs: old_changeset.txs,
-                txouts: old_changeset.txouts,
-                last_seen: old_changeset.last_seen,
-                anchors: old_changeset
-                    .anchors
-                    .into_iter()
-                    .map(|(height_anchor, txid)| {
-                        let confirmation_height = height_anchor.confirmation_height;
-                        let confirmation_time = height_to_time[&confirmation_height];
-                        let time_anchor = ConfirmationTimeHeightAnchor {
-                            anchor_block: height_anchor.anchor_block,
-                            confirmation_height,
-                            confirmation_time,
-                        };
-                        (time_anchor, txid)
-                    })
-                    .collect(),
-            }
-        };
-
-        let mut new_graph = TxGraph::default();
-        new_graph.apply_changeset(graph_changeset);
-        Ok(new_graph)
-    }
-}
-
-/// Combination of chain and transactions updates from electrum
-///
-/// We have to update the chain and the txids at the same time since we anchor the txids to
-/// the same chain tip that we check before and after we gather the txids.
-#[derive(Debug)]
-pub struct ElectrumUpdate {
-    /// Chain update
-    pub chain_update: local_chain::Update,
-    /// Transaction updates from electrum
-    pub relevant_txids: RelevantTxids,
-}
-
-/// Trait to extend [`Client`] functionality.
-pub trait ElectrumExt {
-    /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and
-    /// returns updates for [`bdk_chain`] data structures.
-    ///
-    /// - `prev_tip`: the most recent blockchain tip present locally
-    /// - `keychain_spks`: keychains that we want to scan transactions for
-    ///
-    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
-    /// transactions. `batch_size` specifies the max number of script pubkeys to request for in a
-    /// single batch request.
-    fn full_scan<K: Ord + Clone>(
-        &self,
-        prev_tip: CheckPoint,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        stop_gap: usize,
-        batch_size: usize,
-    ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error>;
-
-    /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified
-    /// and returns updates for [`bdk_chain`] data structures.
-    ///
-    /// - `prev_tip`: the most recent blockchain tip present locally
-    /// - `misc_spks`: an iterator of scripts we want to sync transactions for
-    /// - `txids`: transactions for which we want updated [`Anchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to include in the update
-    ///
-    /// `batch_size` specifies the max number of script pubkeys to request for in a single batch
-    /// request.
-    ///
-    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
-    /// may include scripts that have been used, use [`full_scan`] with the keychain.
-    ///
-    /// [`full_scan`]: ElectrumExt::full_scan
-    fn sync(
-        &self,
-        prev_tip: CheckPoint,
-        misc_spks: impl IntoIterator<Item = ScriptBuf>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate, Error>;
-}
-
-impl<A: ElectrumApi> ElectrumExt for A {
-    fn full_scan<K: Ord + Clone>(
-        &self,
-        prev_tip: CheckPoint,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, ScriptBuf)>>,
-        stop_gap: usize,
-        batch_size: usize,
-    ) -> Result<(ElectrumUpdate, BTreeMap<K, u32>), Error> {
-        let mut request_spks = keychain_spks
-            .into_iter()
-            .map(|(k, s)| (k.clone(), s.into_iter()))
-            .collect::<BTreeMap<K, _>>();
-        let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();
-
-        let (electrum_update, keychain_update) = loop {
-            let (tip, _) = construct_update_tip(self, prev_tip.clone())?;
-            let mut relevant_txids = RelevantTxids::default();
-            let cps = tip
-                .iter()
-                .take(10)
-                .map(|cp| (cp.height(), cp))
-                .collect::<BTreeMap<u32, CheckPoint>>();
-
-            if !request_spks.is_empty() {
-                if !scanned_spks.is_empty() {
-                    scanned_spks.append(&mut populate_with_spks(
-                        self,
-                        &cps,
-                        &mut relevant_txids,
-                        &mut scanned_spks
-                            .iter()
-                            .map(|(i, (spk, _))| (i.clone(), spk.clone())),
-                        stop_gap,
-                        batch_size,
-                    )?);
-                }
-                for (keychain, keychain_spks) in &mut request_spks {
-                    scanned_spks.extend(
-                        populate_with_spks(
-                            self,
-                            &cps,
-                            &mut relevant_txids,
-                            keychain_spks,
-                            stop_gap,
-                            batch_size,
-                        )?
-                        .into_iter()
-                        .map(|(spk_i, spk)| ((keychain.clone(), spk_i), spk)),
-                    );
-                }
-            }
-
-            // check for reorgs during scan process
-            let server_blockhash = self.block_header(tip.height() as usize)?.block_hash();
-            if tip.hash() != server_blockhash {
-                continue; // reorg
-            }
-
-            let chain_update = local_chain::Update {
-                tip,
-                introduce_older_blocks: true,
-            };
-
-            let keychain_update = request_spks
-                .into_keys()
-                .filter_map(|k| {
-                    scanned_spks
-                        .range((k.clone(), u32::MIN)..=(k.clone(), u32::MAX))
-                        .rev()
-                        .find(|(_, (_, active))| *active)
-                        .map(|((_, i), _)| (k, *i))
-                })
-                .collect::<BTreeMap<_, _>>();
-
-            break (
-                ElectrumUpdate {
-                    chain_update,
-                    relevant_txids,
-                },
-                keychain_update,
-            );
-        };
-
-        Ok((electrum_update, keychain_update))
-    }
-
-    fn sync(
-        &self,
-        prev_tip: CheckPoint,
-        misc_spks: impl IntoIterator<Item = ScriptBuf>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate, Error> {
-        let spk_iter = misc_spks
-            .into_iter()
-            .enumerate()
-            .map(|(i, spk)| (i as u32, spk));
-
-        let (mut electrum_update, _) = self.full_scan(
-            prev_tip.clone(),
-            [((), spk_iter)].into(),
-            usize::MAX,
-            batch_size,
-        )?;
-
-        let (tip, _) = construct_update_tip(self, prev_tip)?;
-        let cps = tip
-            .iter()
-            .take(10)
-            .map(|cp| (cp.height(), cp))
-            .collect::<BTreeMap<u32, CheckPoint>>();
-
-        populate_with_txids(self, &cps, &mut electrum_update.relevant_txids, txids)?;
-
-        let _txs =
-            populate_with_outpoints(self, &cps, &mut electrum_update.relevant_txids, outpoints)?;
-
-        Ok(electrum_update)
-    }
-}
-
-/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
-fn construct_update_tip(
-    client: &impl ElectrumApi,
-    prev_tip: CheckPoint,
-) -> Result<(CheckPoint, Option<u32>), Error> {
-    let HeaderNotification { height, .. } = client.block_headers_subscribe()?;
-    let new_tip_height = height as u32;
-
-    // If electrum returns a tip height that is lower than our previous tip, then checkpoints do
-    // not need updating. We just return the previous tip and use that as the point of agreement.
-    if new_tip_height < prev_tip.height() {
-        return Ok((prev_tip.clone(), Some(prev_tip.height())));
-    }
-
-    // Atomically fetch the latest `CHAIN_SUFFIX_LENGTH` count of blocks from Electrum. We use this
-    // to construct our checkpoint update.
-    let mut new_blocks = {
-        let start_height = new_tip_height.saturating_sub(CHAIN_SUFFIX_LENGTH - 1);
-        let hashes = client
-            .block_headers(start_height as _, CHAIN_SUFFIX_LENGTH as _)?
-            .headers
-            .into_iter()
-            .map(|h| h.block_hash());
-        (start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()
-    };
-
-    // Find the "point of agreement" (if any).
-    let agreement_cp = {
-        let mut agreement_cp = Option::<CheckPoint>::None;
-        for cp in prev_tip.iter() {
-            let cp_block = cp.block_id();
-            let hash = match new_blocks.get(&cp_block.height) {
-                Some(&hash) => hash,
-                None => {
-                    assert!(
-                        new_tip_height >= cp_block.height,
-                        "already checked that electrum's tip cannot be smaller"
-                    );
-                    let hash = client.block_header(cp_block.height as _)?.block_hash();
-                    new_blocks.insert(cp_block.height, hash);
-                    hash
-                }
-            };
-            if hash == cp_block.hash {
-                agreement_cp = Some(cp);
-                break;
-            }
-        }
-        agreement_cp
-    };
-
-    let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);
-
-    let new_tip = new_blocks
-        .into_iter()
-        // Prune `new_blocks` to only include blocks that are actually new.
-        .filter(|(height, _)| Some(*height) > agreement_height)
-        .map(|(height, hash)| BlockId { height, hash })
-        .fold(agreement_cp, |prev_cp, block| {
-            Some(match prev_cp {
-                Some(cp) => cp.push(block).expect("must extend checkpoint"),
-                None => CheckPoint::new(block),
-            })
-        })
-        .expect("must have at least one checkpoint");
-
-    Ok((new_tip, agreement_height))
-}
-
-/// A [tx status] comprises of a concatenation of `tx_hash:height:`s. We transform a single one of
-/// these concatenations into a [`ConfirmationHeightAnchor`] if possible.
-///
-/// We use the lowest possible checkpoint as the anchor block (from `cps`). If an anchor block
-/// cannot be found, or the transaction is unconfirmed, [`None`] is returned.
-///
-/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
-fn determine_tx_anchor(
-    cps: &BTreeMap<u32, CheckPoint>,
-    raw_height: i32,
-    txid: Txid,
-) -> Option<ConfirmationHeightAnchor> {
-    // The electrum API has a weird quirk where an unconfirmed transaction is presented with a
-    // height of 0. To avoid invalid representation in our data structures, we manually set
-    // transactions residing in the genesis block to have height 0, then interpret a height of 0 as
-    // unconfirmed for all other transactions.
-    if txid
-        == Txid::from_str("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
-            .expect("must deserialize genesis coinbase txid")
-    {
-        let anchor_block = cps.values().next()?.block_id();
-        return Some(ConfirmationHeightAnchor {
-            anchor_block,
-            confirmation_height: 0,
-        });
-    }
-    match raw_height {
-        h if h <= 0 => {
-            debug_assert!(h == 0 || h == -1, "unexpected height ({}) from electrum", h);
-            None
-        }
-        h => {
-            let h = h as u32;
-            let anchor_block = cps.range(h..).next().map(|(_, cp)| cp.block_id())?;
-            if h > anchor_block.height {
-                None
-            } else {
-                Some(ConfirmationHeightAnchor {
-                    anchor_block,
-                    confirmation_height: h,
-                })
-            }
-        }
-    }
-}
-
-fn populate_with_outpoints(
-    client: &impl ElectrumApi,
-    cps: &BTreeMap<u32, CheckPoint>,
-    relevant_txids: &mut RelevantTxids,
-    outpoints: impl IntoIterator<Item = OutPoint>,
-) -> Result<HashMap<Txid, Transaction>, Error> {
-    let mut full_txs = HashMap::new();
-    for outpoint in outpoints {
-        let txid = outpoint.txid;
-        let tx = client.transaction_get(&txid)?;
-        debug_assert_eq!(tx.txid(), txid);
-        let txout = match tx.output.get(outpoint.vout as usize) {
-            Some(txout) => txout,
-            None => continue,
-        };
-        // attempt to find the following transactions (alongside their chain positions), and
-        // add to our sparsechain `update`:
-        let mut has_residing = false; // tx in which the outpoint resides
-        let mut has_spending = false; // tx that spends the outpoint
-        for res in client.script_get_history(&txout.script_pubkey)? {
-            if has_residing && has_spending {
-                break;
-            }
-
-            if res.tx_hash == txid {
-                if has_residing {
-                    continue;
-                }
-                has_residing = true;
-                full_txs.insert(res.tx_hash, tx.clone());
-            } else {
-                if has_spending {
-                    continue;
-                }
-                let res_tx = match full_txs.get(&res.tx_hash) {
-                    Some(tx) => tx,
-                    None => {
-                        let res_tx = client.transaction_get(&res.tx_hash)?;
-                        full_txs.insert(res.tx_hash, res_tx);
-                        full_txs.get(&res.tx_hash).expect("just inserted")
-                    }
-                };
-                has_spending = res_tx
-                    .input
-                    .iter()
-                    .any(|txin| txin.previous_output == outpoint);
-                if !has_spending {
-                    continue;
-                }
-            };
-
-            let anchor = determine_tx_anchor(cps, res.height, res.tx_hash);
-            let tx_entry = relevant_txids.0.entry(res.tx_hash).or_default();
-            if let Some(anchor) = anchor {
-                tx_entry.insert(anchor);
-            }
-        }
-    }
-    Ok(full_txs)
-}
-
-fn populate_with_txids(
-    client: &impl ElectrumApi,
-    cps: &BTreeMap<u32, CheckPoint>,
-    relevant_txids: &mut RelevantTxids,
-    txids: impl IntoIterator<Item = Txid>,
-) -> Result<(), Error> {
-    for txid in txids {
-        let tx = match client.transaction_get(&txid) {
-            Ok(tx) => tx,
-            Err(electrum_client::Error::Protocol(_)) => continue,
-            Err(other_err) => return Err(other_err),
-        };
-
-        let spk = tx
-            .output
-            .first()
-            .map(|txo| &txo.script_pubkey)
-            .expect("tx must have an output");
-
-        let anchor = match client
-            .script_get_history(spk)?
-            .into_iter()
-            .find(|r| r.tx_hash == txid)
-        {
-            Some(r) => determine_tx_anchor(cps, r.height, txid),
-            None => continue,
-        };
-
-        let tx_entry = relevant_txids.0.entry(txid).or_default();
-        if let Some(anchor) = anchor {
-            tx_entry.insert(anchor);
-        }
-    }
-    Ok(())
-}
-
-fn populate_with_spks<I: Ord + Clone>(
-    client: &impl ElectrumApi,
-    cps: &BTreeMap<u32, CheckPoint>,
-    relevant_txids: &mut RelevantTxids,
-    spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
-    stop_gap: usize,
-    batch_size: usize,
-) -> Result<BTreeMap<I, (ScriptBuf, bool)>, Error> {
-    let mut unused_spk_count = 0_usize;
-    let mut scanned_spks = BTreeMap::new();
-
-    loop {
-        let spks = (0..batch_size)
-            .map_while(|_| spks.next())
-            .collect::<Vec<_>>();
-        if spks.is_empty() {
-            return Ok(scanned_spks);
-        }
-
-        let spk_histories =
-            client.batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?;
-
-        for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {
-            if spk_history.is_empty() {
-                scanned_spks.insert(spk_index, (spk, false));
-                unused_spk_count += 1;
-                if unused_spk_count > stop_gap {
-                    return Ok(scanned_spks);
-                }
-                continue;
-            } else {
-                scanned_spks.insert(spk_index, (spk, true));
-                unused_spk_count = 0;
-            }
-
-            for tx in spk_history {
-                let tx_entry = relevant_txids.0.entry(tx.tx_hash).or_default();
-                if let Some(anchor) = determine_tx_anchor(cps, tx.height, tx.tx_hash) {
-                    tx_entry.insert(anchor);
-                }
-            }
-        }
-    }
-}

crates/electrum/src/lib.rs

@@ -1,30 +1,22 @@
 //! This crate is used for updating structures of [`bdk_chain`] with data from an Electrum server.
 //!
-//! The two primary methods are [`ElectrumExt::sync`] and [`ElectrumExt::full_scan`]. In most cases
-//! [`ElectrumExt::sync`] is used to sync the transaction histories of scripts that the application
+//! The two primary methods are [`BdkElectrumClient::sync`] and [`BdkElectrumClient::full_scan`]. In most cases
+//! [`BdkElectrumClient::sync`] is used to sync the transaction histories of scripts that the application
 //! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
-//! has shown a user. [`ElectrumExt::full_scan`] is meant to be used when importing or restoring a
+//! has shown a user. [`BdkElectrumClient::full_scan`] is meant to be used when importing or restoring a
 //! keychain where the range of possibly used scripts is not known. In this case it is necessary to
 //! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
 //! sync or full scan the user receives relevant blockchain data and output updates for
-//! [`bdk_chain`] including [`RelevantTxids`].
-//!
-//! The [`RelevantTxids`] only includes `txid`s and not full transactions. The caller is responsible
-//! for obtaining full transactions before applying new data to their [`bdk_chain`]. This can be
-//! done with these steps:
-//!
-//! 1. Determine which full transactions are missing. Use [`RelevantTxids::missing_full_txs`].
-//!
-//! 2. Obtaining the full transactions. To do this via electrum use [`ElectrumApi::batch_transaction_get`].
+//! [`bdk_chain`].
 //!
 //! Refer to [`example_electrum`] for a complete example.
 //!
-//! [`ElectrumApi::batch_transaction_get`]: electrum_client::ElectrumApi::batch_transaction_get
 //! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum
 
 #![warn(missing_docs)]
 
-mod electrum_ext;
+mod bdk_electrum_client;
+pub use bdk_electrum_client::*;
+
 pub use bdk_chain;
 pub use electrum_client;
-pub use electrum_ext::*;

crates/electrum/tests/test_electrum.rs

@@ -0,0 +1,437 @@
+use bdk_chain::{
+    bitcoin::{hashes::Hash, Address, Amount, ScriptBuf, Txid, WScriptHash},
+    local_chain::LocalChain,
+    spk_client::{FullScanRequest, SyncRequest},
+    spk_txout::SpkTxOutIndex,
+    Balance, ConfirmationBlockTime, IndexedTxGraph,
+};
+use bdk_electrum::BdkElectrumClient;
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+use std::collections::{BTreeSet, HashSet};
+use std::str::FromStr;
+
+fn get_balance(
+    recv_chain: &LocalChain,
+    recv_graph: &IndexedTxGraph<ConfirmationBlockTime, SpkTxOutIndex<()>>,
+) -> anyhow::Result<Balance> {
+    let chain_tip = recv_chain.tip().block_id();
+    let outpoints = recv_graph.index.outpoints().clone();
+    let balance = recv_graph
+        .graph()
+        .balance(recv_chain, chain_tip, outpoints, |_, _| true);
+    Ok(balance)
+}
+
+#[test]
+pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+
+    let receive_address0 =
+        Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
+    let receive_address1 =
+        Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked();
+
+    let misc_spks = [
+        receive_address0.script_pubkey(),
+        receive_address1.script_pubkey(),
+    ];
+
+    let _block_hashes = env.mine_blocks(101, None)?;
+    let txid1 = env.bitcoind.client.send_to_address(
+        &receive_address1,
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let txid2 = env.bitcoind.client.send_to_address(
+        &receive_address0,
+        Amount::from_sat(20000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    env.mine_blocks(1, None)?;
+    env.wait_until_electrum_sees_block()?;
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    let sync_update = {
+        let request = SyncRequest::from_chain_tip(cp_tip.clone()).set_spks(misc_spks);
+        client.sync(request, 1, true)?
+    };
+
+    assert!(
+        {
+            let update_cps = sync_update
+                .chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            let superset_cps = cp_tip
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            superset_cps.is_superset(&update_cps)
+        },
+        "update should not alter original checkpoint tip since we already started with all checkpoints",
+    );
+
+    let graph_update = sync_update.graph_update;
+    // Check to see if we have the floating txouts available from our two created transactions'
+    // previous outputs in order to calculate transaction fees.
+    for tx in graph_update.full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transactions' previous outputs.
+        let fee = graph_update.calculate_fee(&tx.tx).expect("Fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_unsigned()
+            .expect("valid `Amount`");
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    let mut graph_update_txids: Vec<Txid> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    graph_update_txids.sort();
+    let mut expected_txids = vec![txid1, txid2];
+    expected_txids.sort();
+    assert_eq!(graph_update_txids, expected_txids);
+
+    Ok(())
+}
+
+/// Test the bounds of the address scan depending on the `stop_gap`.
+#[test]
+pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+    let _block_hashes = env.mine_blocks(101, None)?;
+
+    // Now let's test the gap limit. First of all get a chain of 10 addresses.
+    let addresses = [
+        "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4",
+        "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30",
+        "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g",
+        "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww",
+        "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu",
+        "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh",
+        "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2",
+        "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8",
+        "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef",
+        "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk",
+    ];
+    let addresses: Vec<_> = addresses
+        .into_iter()
+        .map(|s| Address::from_str(s).unwrap().assume_checked())
+        .collect();
+    let spks: Vec<_> = addresses
+        .iter()
+        .enumerate()
+        .map(|(i, addr)| (i as u32, addr.script_pubkey()))
+        .collect();
+
+    // Then receive coins on the 4th address.
+    let txid_4th_addr = env.bitcoind.client.send_to_address(
+        &addresses[3],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    env.mine_blocks(1, None)?;
+    env.wait_until_electrum_sees_block()?;
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    // A scan with a stop_gap of 3 won't find the transaction, but a scan with a gap limit of 4
+    // will.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 3, 1, false)?
+    };
+    assert!(full_scan_update.graph_update.full_txs().next().is_none());
+    assert!(full_scan_update.last_active_indices.is_empty());
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 4, 1, false)?
+    };
+    assert_eq!(
+        full_scan_update
+            .graph_update
+            .full_txs()
+            .next()
+            .unwrap()
+            .txid,
+        txid_4th_addr
+    );
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+
+    // Now receive a coin on the last address.
+    let txid_last_addr = env.bitcoind.client.send_to_address(
+        &addresses[addresses.len() - 1],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    env.mine_blocks(1, None)?;
+    env.wait_until_electrum_sees_block()?;
+
+    // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will.
+    // The last active indice won't be updated in the first case but will in the second one.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 5, 1, false)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 1);
+    assert!(txs.contains(&txid_4th_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 6, 1, false)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 2);
+    assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 9);
+
+    Ok(())
+}
+
+/// Ensure that [`ElectrumExt`] can sync properly.
+///
+/// 1. Mine 101 blocks.
+/// 2. Send a tx.
+/// 3. Mine extra block to confirm sent tx.
+/// 4. Check [`Balance`] to ensure tx is confirmed.
+#[test]
+fn scan_detects_confirmed_tx() -> anyhow::Result<()> {
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+
+    // Setup addresses.
+    let addr_to_mine = env
+        .bitcoind
+        .client
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?;
+
+    // Setup receiver.
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.bitcoind.client.get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<ConfirmationBlockTime, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // Mine some blocks.
+    env.mine_blocks(101, Some(addr_to_mine))?;
+
+    // Create transaction that is tracked by our receiver.
+    env.send(&addr_to_track, SEND_AMOUNT)?;
+
+    // Mine a block to confirm sent tx.
+    env.mine_blocks(1, None)?;
+
+    // Sync up to tip.
+    env.wait_until_electrum_sees_block()?;
+    let update = client.sync(
+        SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks(core::iter::once(spk_to_track)),
+        5,
+        true,
+    )?;
+
+    let _ = recv_chain
+        .apply_update(update.chain_update)
+        .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+    let _ = recv_graph.apply_update(update.graph_update);
+
+    // Check to see if tx is confirmed.
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT,
+            ..Balance::default()
+        },
+    );
+
+    for tx in recv_graph.graph().full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transaction's previous outputs.
+        let fee = recv_graph
+            .graph()
+            .calculate_fee(&tx.tx)
+            .expect("fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_unsigned()
+            .expect("valid `Amount`");
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    Ok(())
+}
+
+/// Ensure that confirmed txs that are reorged become unconfirmed.
+///
+/// 1. Mine 101 blocks.
+/// 2. Mine 8 blocks with a confirmed tx in each.
+/// 3. Perform 8 separate reorgs on each block with a confirmed tx.
+/// 4. Check [`Balance`] after each reorg to ensure unconfirmed amount is correct.
+#[test]
+fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
+    const REORG_COUNT: usize = 8;
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+
+    // Setup addresses.
+    let addr_to_mine = env
+        .bitcoind
+        .client
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?;
+
+    // Setup receiver.
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.bitcoind.client.get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<ConfirmationBlockTime, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // Mine some blocks.
+    env.mine_blocks(101, Some(addr_to_mine))?;
+
+    // Create transactions that are tracked by our receiver.
+    let mut txids = vec![];
+    let mut hashes = vec![];
+    for _ in 0..REORG_COUNT {
+        txids.push(env.send(&addr_to_track, SEND_AMOUNT)?);
+        hashes.extend(env.mine_blocks(1, None)?);
+    }
+
+    // Sync up to tip.
+    env.wait_until_electrum_sees_block()?;
+    let update = client.sync(
+        SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]),
+        5,
+        false,
+    )?;
+
+    let _ = recv_chain
+        .apply_update(update.chain_update)
+        .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+    let _ = recv_graph.apply_update(update.graph_update.clone());
+
+    // Retain a snapshot of all anchors before reorg process.
+    let initial_anchors = update.graph_update.all_anchors();
+    let anchors: Vec<_> = initial_anchors.iter().cloned().collect();
+    assert_eq!(anchors.len(), REORG_COUNT);
+    for i in 0..REORG_COUNT {
+        let (anchor, txid) = anchors[i];
+        assert_eq!(anchor.block_id.hash, hashes[i]);
+        assert_eq!(txid, txids[i]);
+    }
+
+    // Check if initial balance is correct.
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT * REORG_COUNT as u64,
+            ..Balance::default()
+        },
+        "initial balance must be correct",
+    );
+
+    // Perform reorgs with different depths.
+    for depth in 1..=REORG_COUNT {
+        env.reorg_empty_blocks(depth)?;
+
+        env.wait_until_electrum_sees_block()?;
+        let update = client.sync(
+            SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]),
+            5,
+            false,
+        )?;
+
+        let _ = recv_chain
+            .apply_update(update.chain_update)
+            .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+
+        // Check that no new anchors are added during current reorg.
+        assert!(initial_anchors.is_superset(update.graph_update.all_anchors()));
+        let _ = recv_graph.apply_update(update.graph_update);
+
+        assert_eq!(
+            get_balance(&recv_chain, &recv_graph)?,
+            Balance {
+                confirmed: SEND_AMOUNT * (REORG_COUNT - depth) as u64,
+                ..Balance::default()
+            },
+            "reorg_count: {}",
+            depth,
+        );
+    }
+
+    Ok(())
+}

crates/esplora/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_esplora"
-version = "0.8.0"
+version = "0.16.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,23 +12,23 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.10.0", default-features = false }
-esplora-client = { version = "0.6.0", default-features = false }
+bdk_chain = { path = "../chain", version = "0.17.0", default-features = false }
+esplora-client = { version = "0.8.0", default-features = false }
 async-trait = { version = "0.1.66", optional = true }
 futures = { version = "0.3.26", optional = true }
 
-# use these dependencies if you need to enable their /no-std features
-bitcoin = { version = "0.30.0", optional = true, default-features = false }
-miniscript = { version = "10.0.0", optional = true, default-features = false }
+bitcoin = { version = "0.32.0", optional = true, default-features = false }
+miniscript = { version = "12.0.0", optional = true, default-features = false }
 
-[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies]
-electrsd = { version= "0.25.0", features = ["bitcoind_25_0", "esplora_a33e97e1", "legacy"] }
+[dev-dependencies]
+bdk_testenv = { path = "../testenv", default-features = false }
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
 
 [features]
-default = ["std", "async-https", "blocking"]
-std = ["bdk_chain/std"]
+default = ["std", "async-https", "blocking-https-rustls"]
+std = ["bdk_chain/std", "miniscript?/std"]
 async = ["async-trait", "futures", "esplora-client/async"]
 async-https = ["async", "esplora-client/async-https"]
 async-https-rustls = ["async", "esplora-client/async-https-rustls"]
 blocking = ["esplora-client/blocking"]
+blocking-https-rustls = ["esplora-client/blocking-https-rustls"]

crates/esplora/src/async_ext.rs

@@ -1,12 +1,15 @@
+use std::collections::BTreeSet;
+
 use async_trait::async_trait;
-use bdk_chain::collections::btree_map;
+use bdk_chain::spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult};
 use bdk_chain::{
     bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid},
     collections::BTreeMap,
-    local_chain::{self, CheckPoint},
-    BlockId, ConfirmationTimeHeightAnchor, TxGraph,
+    local_chain::CheckPoint,
+    BlockId, ConfirmationBlockTime, TxGraph,
 };
-use esplora_client::TxStatus;
+use bdk_chain::{Anchor, Indexed};
+use esplora_client::{Amount, TxStatus};
 use futures::{stream::FuturesOrdered, TryStreamExt};
 
 use crate::anchor_from_status;
@@ -22,53 +25,40 @@ type Error = Box<esplora_client::Error>;
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait EsploraAsyncExt {
-    /// Prepare a [`LocalChain`] update with blocks fetched from Esplora.
+    /// Scan keychain scripts for transactions against Esplora, returning an update that can be
+    /// applied to the receiving structures.
     ///
-    /// * `local_tip` is the previous tip of [`LocalChain::tip`].
-    /// * `request_heights` is the block heights that we are interested in fetching from Esplora.
+    /// - `request`: struct with data required to perform a spk-based blockchain client full scan,
+    ///              see [`FullScanRequest`]
     ///
-    /// The result of this method can be applied to [`LocalChain::apply_update`].
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no
+    /// associated transactions. `parallel_requests` specifies the max number of HTTP requests to
+    /// make in parallel.
     ///
-    /// ## Consistency
+    /// ## Note
     ///
-    /// The chain update returned is guaranteed to be consistent as long as there is not a *large* re-org
-    /// during the call. The size of re-org we can tollerate is server dependent but will be at
-    /// least 10.
+    /// `stop_gap` is defined as "the maximum number of consecutive unused addresses".
+    /// For example, with a `stop_gap` of  3, `full_scan` will keep scanning
+    /// until it encounters 3 consecutive script pubkeys with no associated transactions.
     ///
-    /// [`LocalChain`]: bdk_chain::local_chain::LocalChain
-    /// [`LocalChain::tip`]: bdk_chain::local_chain::LocalChain::tip
-    /// [`LocalChain::apply_update`]: bdk_chain::local_chain::LocalChain::apply_update
-    async fn update_local_chain(
-        &self,
-        local_tip: CheckPoint,
-        request_heights: impl IntoIterator<IntoIter = impl Iterator<Item = u32> + Send> + Send,
-    ) -> Result<local_chain::Update, Error>;
-
-    /// Full scan the keychain scripts specified with the blockchain (via an Esplora client) and
-    /// returns a [`TxGraph`] and a map of last active indices.
+    /// This follows the same approach as other Bitcoin-related software,
+    /// such as [Electrum](https://electrum.readthedocs.io/en/latest/faq.html#what-is-the-gap-limit),
+    /// [BTCPay Server](https://docs.btcpayserver.org/FAQ/Wallet/#the-gap-limit-problem),
+    /// and [Sparrow](https://www.sparrowwallet.com/docs/faq.html#ive-restored-my-wallet-but-some-of-my-funds-are-missing).
     ///
-    /// * `keychain_spks`: keychains that we want to scan transactions for
-    ///
-    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
-    /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
-    /// parallel.
+    /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1.
     async fn full_scan<K: Ord + Clone + Send>(
         &self,
-        keychain_spks: BTreeMap<
-            K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
-        >,
+        request: FullScanRequest<K>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error>;
+    ) -> Result<FullScanResult<K>, Error>;
 
     /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
     /// specified and return a [`TxGraph`].
     ///
-    /// * `misc_spks`: scripts that we want to sync transactions for
-    /// * `txids`: transactions for which we want updated [`ConfirmationTimeHeightAnchor`]s
-    /// * `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to include in the update
+    /// - `request`: struct with data required to perform a spk-based blockchain client sync, see
+    ///              [`SyncRequest`]
     ///
     /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
     /// may include scripts that have been used, use [`full_scan`] with the keychain.
@@ -76,206 +66,210 @@ pub trait EsploraAsyncExt {
     /// [`full_scan`]: EsploraAsyncExt::full_scan
     async fn sync(
         &self,
-        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        request: SyncRequest,
         parallel_requests: usize,
-    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error>;
+    ) -> Result<SyncResult, Error>;
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl EsploraAsyncExt for esplora_client::AsyncClient {
-    async fn update_local_chain(
-        &self,
-        local_tip: CheckPoint,
-        request_heights: impl IntoIterator<IntoIter = impl Iterator<Item = u32> + Send> + Send,
-    ) -> Result<local_chain::Update, Error> {
-        // Fetch latest N (server dependent) blocks from Esplora. The server guarantees these are
-        // consistent.
-        let mut fetched_blocks = self
-            .get_blocks(None)
-            .await?
-            .into_iter()
-            .map(|b| (b.time.height, b.id))
-            .collect::<BTreeMap<u32, BlockHash>>();
-        let new_tip_height = fetched_blocks
-            .keys()
-            .last()
-            .copied()
-            .expect("must have atleast one block");
-
-        // Fetch blocks of heights that the caller is interested in, skipping blocks that are
-        // already fetched when constructing `fetched_blocks`.
-        for height in request_heights {
-            // do not fetch blocks higher than remote tip
-            if height > new_tip_height {
-                continue;
-            }
-            // only fetch what is missing
-            if let btree_map::Entry::Vacant(entry) = fetched_blocks.entry(height) {
-                // ❗The return value of `get_block_hash` is not strictly guaranteed to be consistent
-                // with the chain at the time of `get_blocks` above (there could have been a deep
-                // re-org). Since `get_blocks` returns 10 (or so) blocks we are assuming that it's
-                // not possible to have a re-org deeper than that.
-                entry.insert(self.get_block_hash(height).await?);
-            }
-        }
-
-        // Ensure `fetched_blocks` can create an update that connects with the original chain by
-        // finding a "Point of Agreement".
-        for (height, local_hash) in local_tip.iter().map(|cp| (cp.height(), cp.hash())) {
-            if height > new_tip_height {
-                continue;
-            }
-
-            let fetched_hash = match fetched_blocks.entry(height) {
-                btree_map::Entry::Occupied(entry) => *entry.get(),
-                btree_map::Entry::Vacant(entry) => {
-                    *entry.insert(self.get_block_hash(height).await?)
-                }
-            };
-
-            // We have found point of agreement so the update will connect!
-            if fetched_hash == local_hash {
-                break;
-            }
-        }
-
-        Ok(local_chain::Update {
-            tip: CheckPoint::from_block_ids(fetched_blocks.into_iter().map(BlockId::from))
-                .expect("must be in height order"),
-            introduce_older_blocks: true,
-        })
-    }
-
     async fn full_scan<K: Ord + Clone + Send>(
         &self,
-        keychain_spks: BTreeMap<
-            K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, ScriptBuf)> + Send> + Send,
-        >,
+        request: FullScanRequest<K>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
-        type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
-        let parallel_requests = Ord::max(parallel_requests, 1);
-        let mut graph = TxGraph::<ConfirmationTimeHeightAnchor>::default();
-        let mut last_active_indexes = BTreeMap::<K, u32>::new();
-
-        for (keychain, spks) in keychain_spks {
-            let mut spks = spks.into_iter();
-            let mut last_index = Option::<u32>::None;
-            let mut last_active_index = Option::<u32>::None;
-
-            loop {
-                let handles = spks
-                    .by_ref()
-                    .take(parallel_requests)
-                    .map(|(spk_index, spk)| {
-                        let client = self.clone();
-                        async move {
-                            let mut last_seen = None;
-                            let mut spk_txs = Vec::new();
-                            loop {
-                                let txs = client.scripthash_txs(&spk, last_seen).await?;
-                                let tx_count = txs.len();
-                                last_seen = txs.last().map(|tx| tx.txid);
-                                spk_txs.extend(txs);
-                                if tx_count < 25 {
-                                    break Result::<_, Error>::Ok((spk_index, spk_txs));
-                                }
-                            }
-                        }
-                    })
-                    .collect::<FuturesOrdered<_>>();
-
-                if handles.is_empty() {
-                    break;
-                }
-
-                for (index, txs) in handles.try_collect::<Vec<TxsOfSpkIndex>>().await? {
-                    last_index = Some(index);
-                    if !txs.is_empty() {
-                        last_active_index = Some(index);
-                    }
-                    for tx in txs {
-                        let _ = graph.insert_tx(tx.to_tx());
-                        if let Some(anchor) = anchor_from_status(&tx.status) {
-                            let _ = graph.insert_anchor(tx.txid, anchor);
-                        }
-
-                        let previous_outputs = tx.vin.iter().filter_map(|vin| {
-                            let prevout = vin.prevout.as_ref()?;
-                            Some((
-                                OutPoint {
-                                    txid: vin.txid,
-                                    vout: vin.vout,
-                                },
-                                TxOut {
-                                    script_pubkey: prevout.scriptpubkey.clone(),
-                                    value: prevout.value,
-                                },
-                            ))
-                        });
-
-                        for (outpoint, txout) in previous_outputs {
-                            let _ = graph.insert_txout(outpoint, txout);
-                        }
-                    }
-                }
-
-                let last_index = last_index.expect("Must be set since handles wasn't empty.");
-                let past_gap_limit = if let Some(i) = last_active_index {
-                    last_index > i.saturating_add(stop_gap as u32)
-                } else {
-                    last_index >= stop_gap as u32
-                };
-                if past_gap_limit {
-                    break;
-                }
-            }
-
-            if let Some(last_active_index) = last_active_index {
-                last_active_indexes.insert(keychain, last_active_index);
-            }
-        }
-
-        Ok((graph, last_active_indexes))
+    ) -> Result<FullScanResult<K>, Error> {
+        let latest_blocks = fetch_latest_blocks(self).await?;
+        let (graph_update, last_active_indices) = full_scan_for_index_and_graph(
+            self,
+            request.spks_by_keychain,
+            stop_gap,
+            parallel_requests,
+        )
+        .await?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )
+        .await?;
+        Ok(FullScanResult {
+            chain_update,
+            graph_update,
+            last_active_indices,
+        })
     }
 
     async fn sync(
         &self,
-        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        request: SyncRequest,
         parallel_requests: usize,
-    ) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
-        let mut graph = self
-            .full_scan(
-                [(
-                    (),
-                    misc_spks
-                        .into_iter()
-                        .enumerate()
-                        .map(|(i, spk)| (i as u32, spk)),
-                )]
-                .into(),
-                usize::MAX,
-                parallel_requests,
-            )
-            .await
-            .map(|(g, _)| g)?;
+    ) -> Result<SyncResult, Error> {
+        let latest_blocks = fetch_latest_blocks(self).await?;
+        let graph_update = sync_for_index_and_graph(
+            self,
+            request.spks,
+            request.txids,
+            request.outpoints,
+            parallel_requests,
+        )
+        .await?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )
+        .await?;
+        Ok(SyncResult {
+            chain_update,
+            graph_update,
+        })
+    }
+}
+
+/// Fetch latest blocks from Esplora in an atomic call.
+///
+/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks AND
+/// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for
+/// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use
+/// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when
+/// alternating between chain-sources.
+async fn fetch_latest_blocks(
+    client: &esplora_client::AsyncClient,
+) -> Result<BTreeMap<u32, BlockHash>, Error> {
+    Ok(client
+        .get_blocks(None)
+        .await?
+        .into_iter()
+        .map(|b| (b.time.height, b.id))
+        .collect())
+}
+
+/// Used instead of [`esplora_client::BlockingClient::get_block_hash`].
+///
+/// This first checks the previously fetched `latest_blocks` before fetching from Esplora again.
+async fn fetch_block(
+    client: &esplora_client::AsyncClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    height: u32,
+) -> Result<Option<BlockHash>, Error> {
+    if let Some(&hash) = latest_blocks.get(&height) {
+        return Ok(Some(hash));
+    }
+
+    // We avoid fetching blocks higher than previously fetched `latest_blocks` as the local chain
+    // tip is used to signal for the last-synced-up-to-height.
+    let &tip_height = latest_blocks
+        .keys()
+        .last()
+        .expect("must have atleast one entry");
+    if height > tip_height {
+        return Ok(None);
+    }
+
+    Ok(Some(client.get_block_hash(height).await?))
+}
+
+/// Create the [`local_chain::Update`].
+///
+/// We want to have a corresponding checkpoint per anchor height. However, checkpoints fetched
+/// should not surpass `latest_blocks`.
+async fn chain_update<A: Anchor>(
+    client: &esplora_client::AsyncClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    local_tip: &CheckPoint,
+    anchors: &BTreeSet<(A, Txid)>,
+) -> Result<CheckPoint, Error> {
+    let mut point_of_agreement = None;
+    let mut conflicts = vec![];
+    for local_cp in local_tip.iter() {
+        let remote_hash = match fetch_block(client, latest_blocks, local_cp.height()).await? {
+            Some(hash) => hash,
+            None => continue,
+        };
+        if remote_hash == local_cp.hash() {
+            point_of_agreement = Some(local_cp.clone());
+            break;
+        } else {
+            // it is not strictly necessary to include all the conflicted heights (we do need the
+            // first one) but it seems prudent to make sure the updated chain's heights are a
+            // superset of the existing chain after update.
+            conflicts.push(BlockId {
+                height: local_cp.height(),
+                hash: remote_hash,
+            });
+        }
+    }
+
+    let mut tip = point_of_agreement.expect("remote esplora should have same genesis block");
+
+    tip = tip
+        .extend(conflicts.into_iter().rev())
+        .expect("evicted are in order");
+
+    for anchor in anchors {
+        let height = anchor.0.anchor_block().height;
+        if tip.get(height).is_none() {
+            let hash = match fetch_block(client, latest_blocks, height).await? {
+                Some(hash) => hash,
+                None => continue,
+            };
+            tip = tip.insert(BlockId { height, hash });
+        }
+    }
+
+    // insert the most recent blocks at the tip to make sure we update the tip and make the update
+    // robust.
+    for (&height, &hash) in latest_blocks.iter() {
+        tip = tip.insert(BlockId { height, hash });
+    }
+
+    Ok(tip)
+}
+
+/// This performs a full scan to get an update for the [`TxGraph`] and
+/// [`KeychainTxOutIndex`](bdk_chain::indexer::keychain_txout::KeychainTxOutIndex).
+async fn full_scan_for_index_and_graph<K: Ord + Clone + Send>(
+    client: &esplora_client::AsyncClient,
+    keychain_spks: BTreeMap<
+        K,
+        impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send> + Send,
+    >,
+    stop_gap: usize,
+    parallel_requests: usize,
+) -> Result<(TxGraph<ConfirmationBlockTime>, BTreeMap<K, u32>), Error> {
+    type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
+    let parallel_requests = Ord::max(parallel_requests, 1);
+    let mut graph = TxGraph::<ConfirmationBlockTime>::default();
+    let mut last_active_indexes = BTreeMap::<K, u32>::new();
+
+    for (keychain, spks) in keychain_spks {
+        let mut spks = spks.into_iter();
+        let mut last_index = Option::<u32>::None;
+        let mut last_active_index = Option::<u32>::None;
 
-        let mut txids = txids.into_iter();
         loop {
-            let handles = txids
+            let handles = spks
                 .by_ref()
                 .take(parallel_requests)
-                .filter(|&txid| graph.get_tx(txid).is_none())
-                .map(|txid| {
-                    let client = self.clone();
-                    async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) }
+                .map(|(spk_index, spk)| {
+                    let client = client.clone();
+                    async move {
+                        let mut last_seen = None;
+                        let mut spk_txs = Vec::new();
+                        loop {
+                            let txs = client.scripthash_txs(&spk, last_seen).await?;
+                            let tx_count = txs.len();
+                            last_seen = txs.last().map(|tx| tx.txid);
+                            spk_txs.extend(txs);
+                            if tx_count < 25 {
+                                break Result::<_, Error>::Ok((spk_index, spk_txs));
+                            }
+                        }
+                    }
                 })
                 .collect::<FuturesOrdered<_>>();
 
@@ -283,38 +277,314 @@ impl EsploraAsyncExt for esplora_client::AsyncClient {
                 break;
             }
 
-            for (txid, status) in handles.try_collect::<Vec<(Txid, TxStatus)>>().await? {
-                if let Some(anchor) = anchor_from_status(&status) {
-                    let _ = graph.insert_anchor(txid, anchor);
+            for (index, txs) in handles.try_collect::<Vec<TxsOfSpkIndex>>().await? {
+                last_index = Some(index);
+                if !txs.is_empty() {
+                    last_active_index = Some(index);
                 }
+                for tx in txs {
+                    let _ = graph.insert_tx(tx.to_tx());
+                    if let Some(anchor) = anchor_from_status(&tx.status) {
+                        let _ = graph.insert_anchor(tx.txid, anchor);
+                    }
+
+                    let previous_outputs = tx.vin.iter().filter_map(|vin| {
+                        let prevout = vin.prevout.as_ref()?;
+                        Some((
+                            OutPoint {
+                                txid: vin.txid,
+                                vout: vin.vout,
+                            },
+                            TxOut {
+                                script_pubkey: prevout.scriptpubkey.clone(),
+                                value: Amount::from_sat(prevout.value),
+                            },
+                        ))
+                    });
+
+                    for (outpoint, txout) in previous_outputs {
+                        let _ = graph.insert_txout(outpoint, txout);
+                    }
+                }
+            }
+
+            let last_index = last_index.expect("Must be set since handles wasn't empty.");
+            let gap_limit_reached = if let Some(i) = last_active_index {
+                last_index >= i.saturating_add(stop_gap as u32)
+            } else {
+                last_index + 1 >= stop_gap as u32
+            };
+            if gap_limit_reached {
+                break;
             }
         }
 
-        for op in outpoints.into_iter() {
-            if graph.get_tx(op.txid).is_none() {
-                if let Some(tx) = self.get_tx(&op.txid).await? {
-                    let _ = graph.insert_tx(tx);
-                }
-                let status = self.get_tx_status(&op.txid).await?;
-                if let Some(anchor) = anchor_from_status(&status) {
-                    let _ = graph.insert_anchor(op.txid, anchor);
-                }
-            }
+        if let Some(last_active_index) = last_active_index {
+            last_active_indexes.insert(keychain, last_active_index);
+        }
+    }
 
-            if let Some(op_status) = self.get_output_status(&op.txid, op.vout as _).await? {
-                if let Some(txid) = op_status.txid {
-                    if graph.get_tx(txid).is_none() {
-                        if let Some(tx) = self.get_tx(&txid).await? {
-                            let _ = graph.insert_tx(tx);
-                        }
-                        let status = self.get_tx_status(&txid).await?;
-                        if let Some(anchor) = anchor_from_status(&status) {
-                            let _ = graph.insert_anchor(txid, anchor);
-                        }
+    Ok((graph, last_active_indexes))
+}
+
+async fn sync_for_index_and_graph(
+    client: &esplora_client::AsyncClient,
+    misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
+    txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
+    outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+    parallel_requests: usize,
+) -> Result<TxGraph<ConfirmationBlockTime>, Error> {
+    let mut graph = full_scan_for_index_and_graph(
+        client,
+        [(
+            (),
+            misc_spks
+                .into_iter()
+                .enumerate()
+                .map(|(i, spk)| (i as u32, spk)),
+        )]
+        .into(),
+        usize::MAX,
+        parallel_requests,
+    )
+    .await
+    .map(|(g, _)| g)?;
+
+    let mut txids = txids.into_iter();
+    loop {
+        let handles = txids
+            .by_ref()
+            .take(parallel_requests)
+            .filter(|&txid| graph.get_tx(txid).is_none())
+            .map(|txid| {
+                let client = client.clone();
+                async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) }
+            })
+            .collect::<FuturesOrdered<_>>();
+
+        if handles.is_empty() {
+            break;
+        }
+
+        for (txid, status) in handles.try_collect::<Vec<(Txid, TxStatus)>>().await? {
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = graph.insert_anchor(txid, anchor);
+            }
+        }
+    }
+
+    for op in outpoints.into_iter() {
+        if graph.get_tx(op.txid).is_none() {
+            if let Some(tx) = client.get_tx(&op.txid).await? {
+                let _ = graph.insert_tx(tx);
+            }
+            let status = client.get_tx_status(&op.txid).await?;
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = graph.insert_anchor(op.txid, anchor);
+            }
+        }
+
+        if let Some(op_status) = client.get_output_status(&op.txid, op.vout as _).await? {
+            if let Some(txid) = op_status.txid {
+                if graph.get_tx(txid).is_none() {
+                    if let Some(tx) = client.get_tx(&txid).await? {
+                        let _ = graph.insert_tx(tx);
+                    }
+                    let status = client.get_tx_status(&txid).await?;
+                    if let Some(anchor) = anchor_from_status(&status) {
+                        let _ = graph.insert_anchor(txid, anchor);
                     }
                 }
             }
         }
-        Ok(graph)
+    }
+
+    Ok(graph)
+}
+
+#[cfg(test)]
+mod test {
+    use std::{collections::BTreeSet, time::Duration};
+
+    use bdk_chain::{
+        bitcoin::{hashes::Hash, Txid},
+        local_chain::LocalChain,
+        BlockId,
+    };
+    use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+    use esplora_client::Builder;
+
+    use crate::async_ext::{chain_update, fetch_latest_blocks};
+
+    macro_rules! h {
+        ($index:literal) => {{
+            bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes())
+        }};
+    }
+
+    /// Ensure that update does not remove heights (from original), and all anchor heights are included.
+    #[tokio::test]
+    pub async fn test_finalize_chain_update() -> anyhow::Result<()> {
+        struct TestCase<'a> {
+            name: &'a str,
+            /// Initial blockchain height to start the env with.
+            initial_env_height: u32,
+            /// Initial checkpoint heights to start with.
+            initial_cps: &'a [u32],
+            /// The final blockchain height of the env.
+            final_env_height: u32,
+            /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch
+            /// the blockhash from the env.
+            anchors: &'a [(u32, Txid)],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "chain_extends",
+                initial_env_height: 60,
+                initial_cps: &[59, 60],
+                final_env_height: 90,
+                anchors: &[],
+            },
+            TestCase {
+                name: "introduce_older_heights",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 50,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+            TestCase {
+                name: "introduce_older_heights_after_chain_extends",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 100,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("[{}] running test case: {}", i, t.name);
+
+            let env = TestEnv::new()?;
+            let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+            let client = Builder::new(base_url.as_str()).build_async()?;
+
+            // set env to `initial_env_height`
+            if let Some(to_mine) = t
+                .initial_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height().await? < t.initial_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft initial `local_chain`
+            let local_chain = {
+                let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?);
+                // force `chain_update_blocking` to add all checkpoints in `t.initial_cps`
+                let anchors = t
+                    .initial_cps
+                    .iter()
+                    .map(|&height| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            Txid::all_zeros(),
+                        ))
+                    })
+                    .collect::<anyhow::Result<BTreeSet<_>>>()?;
+                let update = chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client).await?,
+                    &chain.tip(),
+                    &anchors,
+                )
+                .await?;
+                chain.apply_update(update)?;
+                chain
+            };
+            println!("local chain height: {}", local_chain.tip().height());
+
+            // extend env chain
+            if let Some(to_mine) = t
+                .final_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height().await? < t.final_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft update
+            let update = {
+                let anchors = t
+                    .anchors
+                    .iter()
+                    .map(|&(height, txid)| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            txid,
+                        ))
+                    })
+                    .collect::<anyhow::Result<_>>()?;
+                chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client).await?,
+                    &local_chain.tip(),
+                    &anchors,
+                )
+                .await?
+            };
+
+            // apply update
+            let mut updated_local_chain = local_chain.clone();
+            updated_local_chain.apply_update(update)?;
+            println!(
+                "updated local chain height: {}",
+                updated_local_chain.tip().height()
+            );
+
+            assert!(
+                {
+                    let initial_heights = local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    let updated_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    updated_heights.is_superset(&initial_heights)
+                },
+                "heights from the initial chain must all be in the updated chain",
+            );
+
+            assert!(
+                {
+                    let exp_anchor_heights = t
+                        .anchors
+                        .iter()
+                        .map(|(h, _)| *h)
+                        .chain(t.initial_cps.iter().copied())
+                        .collect::<BTreeSet<_>>();
+                    let anchor_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    anchor_heights.is_superset(&exp_anchor_heights)
+                },
+                "anchor heights must all be in updated chain",
+            );
+        }
+
+        Ok(())
     }
 }

crates/esplora/src/lib.rs

@@ -16,7 +16,7 @@
 //! [`TxGraph`]: bdk_chain::tx_graph::TxGraph
 //! [`example_esplora`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_esplora
 
-use bdk_chain::{BlockId, ConfirmationTimeHeightAnchor};
+use bdk_chain::{BlockId, ConfirmationBlockTime};
 use esplora_client::TxStatus;
 
 pub use esplora_client;
@@ -31,7 +31,7 @@ mod async_ext;
 #[cfg(feature = "async")]
 pub use async_ext::*;
 
-fn anchor_from_status(status: &TxStatus) -> Option<ConfirmationTimeHeightAnchor> {
+fn anchor_from_status(status: &TxStatus) -> Option<ConfirmationBlockTime> {
     if let TxStatus {
         block_height: Some(height),
         block_hash: Some(hash),
@@ -39,9 +39,8 @@ fn anchor_from_status(status: &TxStatus) -> Option<ConfirmationTimeHeightAnchor>
         ..
     } = status.clone()
     {
-        Some(ConfirmationTimeHeightAnchor {
-            anchor_block: BlockId { height, hash },
-            confirmation_height: height,
+        Some(ConfirmationBlockTime {
+            block_id: BlockId { height, hash },
             confirmation_time: time,
         })
     } else {

crates/esplora/tests/async_ext.rs

@@ -1,68 +1,20 @@
+use bdk_chain::spk_client::{FullScanRequest, SyncRequest};
 use bdk_esplora::EsploraAsyncExt;
-use electrsd::bitcoind::bitcoincore_rpc::RpcApi;
-use electrsd::bitcoind::{self, anyhow, BitcoinD};
-use electrsd::{Conf, ElectrsD};
-use esplora_client::{self, AsyncClient, Builder};
-use std::collections::{BTreeMap, HashSet};
+use esplora_client::{self, Builder};
+use std::collections::{BTreeSet, HashSet};
 use std::str::FromStr;
 use std::thread::sleep;
 use std::time::Duration;
 
-use bdk_chain::bitcoin::{Address, Amount, BlockHash, Txid};
-
-struct TestEnv {
-    bitcoind: BitcoinD,
-    #[allow(dead_code)]
-    electrsd: ElectrsD,
-    client: AsyncClient,
-}
-
-impl TestEnv {
-    fn new() -> Result<Self, anyhow::Error> {
-        let bitcoind_exe =
-            bitcoind::downloaded_exe_path().expect("bitcoind version feature must be enabled");
-        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
-
-        let mut electrs_conf = Conf::default();
-        electrs_conf.http_enabled = true;
-        let electrs_exe =
-            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
-        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &electrs_conf)?;
-
-        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
-        let client = Builder::new(base_url.as_str()).build_async()?;
-
-        Ok(Self {
-            bitcoind,
-            electrsd,
-            client,
-        })
-    }
-
-    fn mine_blocks(
-        &self,
-        count: usize,
-        address: Option<Address>,
-    ) -> anyhow::Result<Vec<BlockHash>> {
-        let coinbase_address = match address {
-            Some(address) => address,
-            None => self
-                .bitcoind
-                .client
-                .get_new_address(None, None)?
-                .assume_checked(),
-        };
-        let block_hashes = self
-            .bitcoind
-            .client
-            .generate_to_address(count as _, &coinbase_address)?;
-        Ok(block_hashes)
-    }
-}
+use bdk_chain::bitcoin::{Address, Amount, Txid};
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
 
 #[tokio::test]
 pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_async()?;
+
     let receive_address0 =
         Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
     let receive_address1 =
@@ -95,26 +47,41 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().await.unwrap() < 102 {
+    while client.get_height().await.unwrap() < 102 {
         sleep(Duration::from_millis(10))
     }
 
-    let graph_update = env
-        .client
-        .sync(
-            misc_spks.into_iter(),
-            vec![].into_iter(),
-            vec![].into_iter(),
-            1,
-        )
-        .await?;
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
 
+    let sync_update = {
+        let request = SyncRequest::from_chain_tip(cp_tip.clone()).set_spks(misc_spks);
+        client.sync(request, 1).await?
+    };
+
+    assert!(
+        {
+            let update_cps = sync_update
+                .chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            let superset_cps = cp_tip
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            superset_cps.is_superset(&update_cps)
+        },
+        "update should not alter original checkpoint tip since we already started with all checkpoints",
+    );
+
+    let graph_update = sync_update.graph_update;
     // Check to see if we have the floating txouts available from our two created transactions'
     // previous outputs in order to calculate transaction fees.
     for tx in graph_update.full_txs() {
         // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
         // floating txouts available from the transactions' previous outputs.
-        let fee = graph_update.calculate_fee(tx.tx).expect("Fee must exist");
+        let fee = graph_update.calculate_fee(&tx.tx).expect("Fee must exist");
 
         // Retrieve the fee in the transaction data from `bitcoind`.
         let tx_fee = env
@@ -125,7 +92,8 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
             .fee
             .expect("Fee must exist")
             .abs()
-            .to_sat() as u64;
+            .to_unsigned()
+            .expect("valid `Amount`");
 
         // Check that the calculated fee matches the fee from the transaction data.
         assert_eq!(fee, tx_fee);
@@ -139,10 +107,12 @@ pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// Test the bounds of the address scan depending on the gap limit.
+/// Test the bounds of the address scan depending on the `stop_gap`.
 #[tokio::test]
-pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
+pub async fn test_async_update_tx_graph_stop_gap() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_async()?;
     let _block_hashes = env.mine_blocks(101, None)?;
 
     // Now let's test the gap limit. First of all get a chain of 10 addresses.
@@ -167,8 +137,6 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         .enumerate()
         .map(|(i, addr)| (i as u32, addr.script_pubkey()))
         .collect();
-    let mut keychains = BTreeMap::new();
-    keychains.insert(0, spks);
 
     // Then receive coins on the 4th address.
     let txid_4th_addr = env.bitcoind.client.send_to_address(
@@ -182,18 +150,37 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().await.unwrap() < 103 {
+    while client.get_height().await.unwrap() < 103 {
         sleep(Duration::from_millis(10))
     }
 
-    // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    // A scan with a gap limit of 3 won't find the transaction, but a scan with a gap limit of 4
     // will.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1).await?;
-    assert!(graph_update.full_txs().next().is_none());
-    assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1).await?;
-    assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
-    assert_eq!(active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 3, 1).await?
+    };
+    assert!(full_scan_update.graph_update.full_txs().next().is_none());
+    assert!(full_scan_update.last_active_indices.is_empty());
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 4, 1).await?
+    };
+    assert_eq!(
+        full_scan_update
+            .graph_update
+            .full_txs()
+            .next()
+            .unwrap()
+            .txid,
+        txid_4th_addr
+    );
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
 
     // Now receive a coin on the last address.
     let txid_last_addr = env.bitcoind.client.send_to_address(
@@ -207,22 +194,38 @@ pub async fn test_async_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().await.unwrap() < 104 {
+    while client.get_height().await.unwrap() < 104 {
         sleep(Duration::from_millis(10))
     }
 
-    // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
+    // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1).await?;
-    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 5, 1).await?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
-    assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1).await?;
-    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 6, 1).await?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
-    assert_eq!(active_indices[&0], 9);
+    assert_eq!(full_scan_update.last_active_indices[&0], 9);
 
     Ok(())
 }

crates/esplora/tests/blocking_ext.rs

@@ -1,98 +1,20 @@
-use bdk_chain::local_chain::LocalChain;
-use bdk_chain::BlockId;
+use bdk_chain::spk_client::{FullScanRequest, SyncRequest};
 use bdk_esplora::EsploraExt;
-use electrsd::bitcoind::bitcoincore_rpc::RpcApi;
-use electrsd::bitcoind::{self, anyhow, BitcoinD};
-use electrsd::{Conf, ElectrsD};
-use esplora_client::{self, BlockingClient, Builder};
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use esplora_client::{self, Builder};
+use std::collections::{BTreeSet, HashSet};
 use std::str::FromStr;
 use std::thread::sleep;
 use std::time::Duration;
 
-use bdk_chain::bitcoin::{Address, Amount, BlockHash, Txid};
-
-macro_rules! h {
-    ($index:literal) => {{
-        bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes())
-    }};
-}
-
-macro_rules! local_chain {
-    [ $(($height:expr, $block_hash:expr)), * ] => {{
-        #[allow(unused_mut)]
-        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect())
-            .expect("chain must have genesis block")
-    }};
-}
-
-struct TestEnv {
-    bitcoind: BitcoinD,
-    #[allow(dead_code)]
-    electrsd: ElectrsD,
-    client: BlockingClient,
-}
-
-impl TestEnv {
-    fn new() -> Result<Self, anyhow::Error> {
-        let bitcoind_exe =
-            bitcoind::downloaded_exe_path().expect("bitcoind version feature must be enabled");
-        let bitcoind = BitcoinD::new(bitcoind_exe).unwrap();
-
-        let mut electrs_conf = Conf::default();
-        electrs_conf.http_enabled = true;
-        let electrs_exe =
-            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
-        let electrsd = ElectrsD::with_conf(electrs_exe, &bitcoind, &electrs_conf)?;
-
-        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
-        let client = Builder::new(base_url.as_str()).build_blocking()?;
-
-        Ok(Self {
-            bitcoind,
-            electrsd,
-            client,
-        })
-    }
-
-    fn reset_electrsd(mut self) -> anyhow::Result<Self> {
-        let mut electrs_conf = Conf::default();
-        electrs_conf.http_enabled = true;
-        let electrs_exe =
-            electrsd::downloaded_exe_path().expect("electrs version feature must be enabled");
-        let electrsd = ElectrsD::with_conf(electrs_exe, &self.bitcoind, &electrs_conf)?;
-
-        let base_url = format!("http://{}", &electrsd.esplora_url.clone().unwrap());
-        let client = Builder::new(base_url.as_str()).build_blocking()?;
-        self.electrsd = electrsd;
-        self.client = client;
-        Ok(self)
-    }
-
-    fn mine_blocks(
-        &self,
-        count: usize,
-        address: Option<Address>,
-    ) -> anyhow::Result<Vec<BlockHash>> {
-        let coinbase_address = match address {
-            Some(address) => address,
-            None => self
-                .bitcoind
-                .client
-                .get_new_address(None, None)?
-                .assume_checked(),
-        };
-        let block_hashes = self
-            .bitcoind
-            .client
-            .generate_to_address(count as _, &coinbase_address)?;
-        Ok(block_hashes)
-    }
-}
+use bdk_chain::bitcoin::{Address, Amount, Txid};
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
 
 #[test]
 pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_blocking();
+
     let receive_address0 =
         Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
     let receive_address1 =
@@ -125,23 +47,41 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().unwrap() < 102 {
+    while client.get_height().unwrap() < 102 {
         sleep(Duration::from_millis(10))
     }
 
-    let graph_update = env.client.sync(
-        misc_spks.into_iter(),
-        vec![].into_iter(),
-        vec![].into_iter(),
-        1,
-    )?;
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
 
+    let sync_update = {
+        let request = SyncRequest::from_chain_tip(cp_tip.clone()).set_spks(misc_spks);
+        client.sync(request, 1)?
+    };
+
+    assert!(
+        {
+            let update_cps = sync_update
+                .chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            let superset_cps = cp_tip
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            superset_cps.is_superset(&update_cps)
+        },
+        "update should not alter original checkpoint tip since we already started with all checkpoints",
+    );
+
+    let graph_update = sync_update.graph_update;
     // Check to see if we have the floating txouts available from our two created transactions'
     // previous outputs in order to calculate transaction fees.
     for tx in graph_update.full_txs() {
         // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
         // floating txouts available from the transactions' previous outputs.
-        let fee = graph_update.calculate_fee(tx.tx).expect("Fee must exist");
+        let fee = graph_update.calculate_fee(&tx.tx).expect("Fee must exist");
 
         // Retrieve the fee in the transaction data from `bitcoind`.
         let tx_fee = env
@@ -152,7 +92,8 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
             .fee
             .expect("Fee must exist")
             .abs()
-            .to_sat() as u64;
+            .to_unsigned()
+            .expect("valid `Amount`");
 
         // Check that the calculated fee matches the fee from the transaction data.
         assert_eq!(fee, tx_fee);
@@ -167,10 +108,12 @@ pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
     Ok(())
 }
 
-/// Test the bounds of the address scan depending on the gap limit.
+/// Test the bounds of the address scan depending on the `stop_gap`.
 #[test]
-pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
+pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> {
     let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_blocking();
     let _block_hashes = env.mine_blocks(101, None)?;
 
     // Now let's test the gap limit. First of all get a chain of 10 addresses.
@@ -195,8 +138,6 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         .enumerate()
         .map(|(i, addr)| (i as u32, addr.script_pubkey()))
         .collect();
-    let mut keychains = BTreeMap::new();
-    keychains.insert(0, spks);
 
     // Then receive coins on the 4th address.
     let txid_4th_addr = env.bitcoind.client.send_to_address(
@@ -210,18 +151,37 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().unwrap() < 103 {
+    while client.get_height().unwrap() < 103 {
         sleep(Duration::from_millis(10))
     }
 
-    // A scan with a gap limit of 2 won't find the transaction, but a scan with a gap limit of 3
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    // A scan with a stop_gap of 3 won't find the transaction, but a scan with a gap limit of 4
     // will.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 2, 1)?;
-    assert!(graph_update.full_txs().next().is_none());
-    assert!(active_indices.is_empty());
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 3, 1)?;
-    assert_eq!(graph_update.full_txs().next().unwrap().txid, txid_4th_addr);
-    assert_eq!(active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 3, 1)?
+    };
+    assert!(full_scan_update.graph_update.full_txs().next().is_none());
+    assert!(full_scan_update.last_active_indices.is_empty());
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 4, 1)?
+    };
+    assert_eq!(
+        full_scan_update
+            .graph_update
+            .full_txs()
+            .next()
+            .unwrap()
+            .txid,
+        txid_4th_addr
+    );
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
 
     // Now receive a coin on the last address.
     let txid_last_addr = env.bitcoind.client.send_to_address(
@@ -235,199 +195,38 @@ pub fn test_update_tx_graph_gap_limit() -> anyhow::Result<()> {
         None,
     )?;
     let _block_hashes = env.mine_blocks(1, None)?;
-    while env.client.get_height().unwrap() < 104 {
+    while client.get_height().unwrap() < 104 {
         sleep(Duration::from_millis(10))
     }
 
-    // A scan with gap limit 4 won't find the second transaction, but a scan with gap limit 5 will.
+    // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will.
     // The last active indice won't be updated in the first case but will in the second one.
-    let (graph_update, active_indices) = env.client.full_scan(keychains.clone(), 4, 1)?;
-    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 5, 1)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
     assert_eq!(txs.len(), 1);
     assert!(txs.contains(&txid_4th_addr));
-    assert_eq!(active_indices[&0], 3);
-    let (graph_update, active_indices) = env.client.full_scan(keychains, 5, 1)?;
-    let txs: HashSet<_> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 6, 1)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
     assert_eq!(txs.len(), 2);
     assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
-    assert_eq!(active_indices[&0], 9);
-
-    Ok(())
-}
-
-#[test]
-fn update_local_chain() -> anyhow::Result<()> {
-    const TIP_HEIGHT: u32 = 50;
-
-    let env = TestEnv::new()?;
-    let blocks = {
-        let bitcoind_client = &env.bitcoind.client;
-        assert_eq!(bitcoind_client.get_block_count()?, 1);
-        [
-            (0, bitcoind_client.get_block_hash(0)?),
-            (1, bitcoind_client.get_block_hash(1)?),
-        ]
-        .into_iter()
-        .chain((2..).zip(env.mine_blocks((TIP_HEIGHT - 1) as usize, None)?))
-        .collect::<BTreeMap<_, _>>()
-    };
-    // so new blocks can be seen by Electrs
-    let env = env.reset_electrsd()?;
-
-    struct TestCase {
-        name: &'static str,
-        chain: LocalChain,
-        request_heights: &'static [u32],
-        exp_update_heights: &'static [u32],
-    }
-
-    let test_cases = [
-        TestCase {
-            name: "request_later_blocks",
-            chain: local_chain![(0, blocks[&0]), (21, blocks[&21])],
-            request_heights: &[22, 25, 28],
-            exp_update_heights: &[21, 22, 25, 28],
-        },
-        TestCase {
-            name: "request_prev_blocks",
-            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (5, blocks[&5])],
-            request_heights: &[4],
-            exp_update_heights: &[4, 5],
-        },
-        TestCase {
-            name: "request_prev_blocks_2",
-            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (10, blocks[&10])],
-            request_heights: &[4, 6],
-            exp_update_heights: &[4, 6, 10],
-        },
-        TestCase {
-            name: "request_later_and_prev_blocks",
-            chain: local_chain![(0, blocks[&0]), (7, blocks[&7]), (11, blocks[&11])],
-            request_heights: &[8, 9, 15],
-            exp_update_heights: &[8, 9, 11, 15],
-        },
-        TestCase {
-            name: "request_tip_only",
-            chain: local_chain![(0, blocks[&0]), (5, blocks[&5]), (49, blocks[&49])],
-            request_heights: &[TIP_HEIGHT],
-            exp_update_heights: &[49],
-        },
-        TestCase {
-            name: "request_nothing",
-            chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, blocks[&23])],
-            request_heights: &[],
-            exp_update_heights: &[23],
-        },
-        TestCase {
-            name: "request_nothing_during_reorg",
-            chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, h!("23"))],
-            request_heights: &[],
-            exp_update_heights: &[13, 23],
-        },
-        TestCase {
-            name: "request_nothing_during_reorg_2",
-            chain: local_chain![
-                (0, blocks[&0]),
-                (21, blocks[&21]),
-                (22, h!("22")),
-                (23, h!("23"))
-            ],
-            request_heights: &[],
-            exp_update_heights: &[21, 22, 23],
-        },
-        TestCase {
-            name: "request_prev_blocks_during_reorg",
-            chain: local_chain![
-                (0, blocks[&0]),
-                (21, blocks[&21]),
-                (22, h!("22")),
-                (23, h!("23"))
-            ],
-            request_heights: &[17, 20],
-            exp_update_heights: &[17, 20, 21, 22, 23],
-        },
-        TestCase {
-            name: "request_later_blocks_during_reorg",
-            chain: local_chain![
-                (0, blocks[&0]),
-                (9, blocks[&9]),
-                (22, h!("22")),
-                (23, h!("23"))
-            ],
-            request_heights: &[25, 27],
-            exp_update_heights: &[9, 22, 23, 25, 27],
-        },
-        TestCase {
-            name: "request_later_blocks_during_reorg_2",
-            chain: local_chain![(0, blocks[&0]), (9, h!("9"))],
-            request_heights: &[10],
-            exp_update_heights: &[0, 9, 10],
-        },
-        TestCase {
-            name: "request_later_and_prev_blocks_during_reorg",
-            chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (9, h!("9"))],
-            request_heights: &[8, 11],
-            exp_update_heights: &[1, 8, 9, 11],
-        },
-    ];
-
-    for (i, t) in test_cases.into_iter().enumerate() {
-        println!("Case {}: {}", i, t.name);
-        let mut chain = t.chain;
-
-        let update = env
-            .client
-            .update_local_chain(chain.tip(), t.request_heights.iter().copied())
-            .map_err(|err| {
-                anyhow::format_err!("[{}:{}] `update_local_chain` failed: {}", i, t.name, err)
-            })?;
-
-        let update_blocks = update
-            .tip
-            .iter()
-            .map(|cp| cp.block_id())
-            .collect::<BTreeSet<_>>();
-
-        let exp_update_blocks = t
-            .exp_update_heights
-            .iter()
-            .map(|&height| {
-                let hash = blocks[&height];
-                BlockId { height, hash }
-            })
-            .chain(
-                // Electrs Esplora `get_block` call fetches 10 blocks which is included in the
-                // update
-                blocks
-                    .range(TIP_HEIGHT - 9..)
-                    .map(|(&height, &hash)| BlockId { height, hash }),
-            )
-            .collect::<BTreeSet<_>>();
-
-        assert_eq!(
-            update_blocks, exp_update_blocks,
-            "[{}:{}] unexpected update",
-            i, t.name
-        );
-
-        let _ = chain
-            .apply_update(update)
-            .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err));
-
-        // all requested heights must exist in the final chain
-        for height in t.request_heights {
-            let exp_blockhash = blocks.get(height).expect("block must exist in bitcoind");
-            assert_eq!(
-                chain.blocks().get(height),
-                Some(exp_blockhash),
-                "[{}:{}] block {}:{} must exist in final chain",
-                i,
-                t.name,
-                height,
-                exp_blockhash
-            );
-        }
-    }
+    assert_eq!(full_scan_update.last_active_indices[&0], 9);
 
     Ok(())
 }

crates/file_store/Cargo.toml

@@ -1,17 +1,17 @@
 [package]
 name = "bdk_file_store"
-version = "0.6.0"
+version = "0.14.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk_file_store"
-description = "A simple append-only flat file implementation of Persist for Bitcoin Dev Kit."
+description = "A simple append-only flat file database for persisting bdk_chain data."
 keywords = ["bitcoin", "persist", "persistence", "bdk", "file"]
 authors = ["Bitcoin Dev Kit Developers"]
 readme = "README.md"
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.10.0", features = [ "serde", "miniscript" ] }
+bdk_chain = { path = "../chain", version = "0.17.0", features = [ "serde", "miniscript" ] }
 bincode = { version = "1" }
 serde = { version = "1", features = ["derive"] }
 

crates/file_store/README.md

@@ -1,10 +1,7 @@
 # BDK File Store
 
-This is a simple append-only flat file implementation of
-[`Persist`](`bdk_chain::Persist`).
+This is a simple append-only flat file database for persisting [`bdk_chain`] changesets.
 
-The main structure is [`Store`](`crate::Store`), which can be used with [`bdk`]'s
-`Wallet` to persist wallet data into a flat file.
+The main structure is [`Store`] which works with any [`bdk_chain`] based changesets to persist data into a flat file.
 
-[`bdk`]: https://docs.rs/bdk/latest
-[`bdk_chain`]: https://docs.rs/bdk_chain/latest
+[`bdk_chain`]:https://docs.rs/bdk_chain/latest/bdk_chain/

crates/file_store/src/store.rs

@@ -1,46 +1,32 @@
+use crate::{bincode_options, EntryIter, FileError, IterError};
+use bdk_chain::Merge;
+use bincode::Options;
 use std::{
-    fmt::Debug,
+    fmt::{self, Debug},
     fs::{File, OpenOptions},
     io::{self, Read, Seek, Write},
     marker::PhantomData,
     path::Path,
 };
 
-use bdk_chain::{Append, PersistBackend};
-use bincode::Options;
-
-use crate::{bincode_options, EntryIter, FileError, IterError};
-
 /// Persists an append-only list of changesets (`C`) to a single file.
-///
-/// The changesets are the results of altering a tracker implementation (`T`).
 #[derive(Debug)]
-pub struct Store<C> {
+pub struct Store<C>
+where
+    C: Sync + Send,
+{
     magic_len: usize,
     db_file: File,
     marker: PhantomData<C>,
 }
 
-impl<C> PersistBackend<C> for Store<C>
-where
-    C: Append + serde::Serialize + serde::de::DeserializeOwned,
-{
-    type WriteError = std::io::Error;
-
-    type LoadError = IterError;
-
-    fn write_changes(&mut self, changeset: &C) -> Result<(), Self::WriteError> {
-        self.append_changeset(changeset)
-    }
-
-    fn load_from_persistence(&mut self) -> Result<Option<C>, Self::LoadError> {
-        self.aggregate_changesets().map_err(|e| e.iter_error)
-    }
-}
-
 impl<C> Store<C>
 where
-    C: Append + serde::Serialize + serde::de::DeserializeOwned,
+    C: Merge
+        + serde::Serialize
+        + serde::de::DeserializeOwned
+        + core::marker::Send
+        + core::marker::Sync,
 {
     /// Create a new [`Store`] file in write-only mode; error if the file exists.
     ///
@@ -64,6 +50,7 @@ where
             .create(true)
             .read(true)
             .write(true)
+            .truncate(true)
             .open(file_path)?;
         f.write_all(magic)?;
         Ok(Self {
@@ -143,7 +130,7 @@ where
     ///
     /// You should usually check the error. In many applications, it may make sense to do a full
     /// wallet scan with a stop-gap after getting an error, since it is likely that one of the
-    /// changesets it was unable to read changed the derivation indices of the tracker.
+    /// changesets was unable to read changes of the derivation indices of a keychain.
     ///
     /// **WARNING**: This method changes the write position of the underlying file. The next
     /// changeset will be written over the erroring entry (or the end of the file if none existed).
@@ -160,7 +147,7 @@ where
                 }
             };
             match &mut changeset {
-                Some(changeset) => changeset.append(next_changeset),
+                Some(changeset) => changeset.merge(next_changeset),
                 changeset => *changeset = Some(next_changeset),
             }
         }
@@ -181,7 +168,7 @@ where
         bincode_options()
             .serialize_into(&mut self.db_file, changeset)
             .map_err(|e| match *e {
-                bincode::ErrorKind::Io(inner) => inner,
+                bincode::ErrorKind::Io(error) => error,
                 unexpected_err => panic!("unexpected bincode error: {}", unexpected_err),
             })?;
 
@@ -211,7 +198,7 @@ impl<C> std::fmt::Display for AggregateChangesetsError<C> {
     }
 }
 
-impl<C: std::fmt::Debug> std::error::Error for AggregateChangesetsError<C> {}
+impl<C: fmt::Debug> std::error::Error for AggregateChangesetsError<C> {}
 
 #[cfg(test)]
 mod test {
@@ -231,9 +218,6 @@ mod test {
 
     type TestChangeSet = BTreeSet<String>;
 
-    #[derive(Debug)]
-    struct TestTracker;
-
     /// Check behavior of [`Store::create_new`] and [`Store::open`].
     #[test]
     fn construct_store() {
@@ -381,7 +365,7 @@ mod test {
                 assert_eq!(
                     err.changeset,
                     changesets.iter().cloned().reduce(|mut acc, cs| {
-                        Append::append(&mut acc, cs);
+                        Merge::merge(&mut acc, cs);
                         acc
                     }),
                     "should recover all changesets that are written in full",
@@ -402,7 +386,7 @@ mod test {
                         .cloned()
                         .chain(core::iter::once(last_changeset.clone()))
                         .reduce(|mut acc, cs| {
-                            Append::append(&mut acc, cs);
+                            Merge::merge(&mut acc, cs);
                             acc
                         }),
                     "should recover all changesets",
@@ -438,13 +422,13 @@ mod test {
                 .take(read_count)
                 .map(|r| r.expect("must read valid changeset"))
                 .fold(TestChangeSet::default(), |mut acc, v| {
-                    Append::append(&mut acc, v);
+                    Merge::merge(&mut acc, v);
                     acc
                 });
             // We write after a short read.
-            db.write_changes(&last_changeset)
+            db.append_changeset(&last_changeset)
                 .expect("last write must succeed");
-            Append::append(&mut exp_aggregation, last_changeset.clone());
+            Merge::merge(&mut exp_aggregation, last_changeset.clone());
             drop(db);
 
             // We open the file again and check whether aggregate changeset is expected.

crates/hwi/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_hwi"
-version = "0.1.0"
+version = "0.4.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -9,5 +9,5 @@ license = "MIT OR Apache-2.0"
 readme = "README.md"
 
 [dependencies]
-bdk = { path = "../bdk" }
-hwi = { version = "0.7.0", features = [ "miniscript"] }
+bdk_wallet = { path = "../wallet", version = "1.0.0-beta.1" }
+hwi = { version = "0.9.0", features = [ "miniscript"] }

crates/hwi/README.md

@@ -0,0 +1,3 @@
+# BDK HWI Signer
+
+This crate contains `HWISigner`, an implementation of a `TransactionSigner` to be used with hardware wallets.

crates/hwi/src/lib.rs

@@ -3,13 +3,14 @@
 //! This crate contains HWISigner, an implementation of a [`TransactionSigner`] to be
 //! used with hardware wallets.
 //! ```no_run
-//! # use bdk::bitcoin::Network;
-//! # use bdk::signer::SignerOrdering;
+//! # use bdk_wallet::bitcoin::Network;
+//! # use bdk_wallet::descriptor::Descriptor;
+//! # use bdk_wallet::signer::SignerOrdering;
 //! # use bdk_hwi::HWISigner;
-//! # use bdk::wallet::AddressIndex::New;
-//! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
+//! # use bdk_wallet::{KeychainKind, SignOptions, Wallet};
 //! # use hwi::HWIClient;
 //! # use std::sync::Arc;
+//! # use std::str::FromStr;
 //! #
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
 //! let mut devices = HWIClient::enumerate()?;
@@ -19,11 +20,7 @@
 //! let first_device = devices.remove(0)?;
 //! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
 //!
-//! # let mut wallet = Wallet::new_no_persist(
-//! #     "",
-//! #     None,
-//! #     Network::Testnet,
-//! # )?;
+//! # let mut wallet = Wallet::create("", "").network(Network::Testnet).create_wallet_no_persist()?;
 //! #
 //! // Adding the hardware signer to the BDK wallet
 //! wallet.add_signer(
@@ -36,7 +33,7 @@
 //! # }
 //! ```
 //!
-//! [`TransactionSigner`]: bdk::wallet::signer::TransactionSigner
+//! [`TransactionSigner`]: bdk_wallet::signer::TransactionSigner
 
 mod signer;
 pub use signer::*;

crates/hwi/src/signer.rs

@@ -1,12 +1,12 @@
-use bdk::bitcoin::bip32::Fingerprint;
-use bdk::bitcoin::psbt::PartiallySignedTransaction;
-use bdk::bitcoin::secp256k1::{All, Secp256k1};
+use bdk_wallet::bitcoin::bip32::Fingerprint;
+use bdk_wallet::bitcoin::secp256k1::{All, Secp256k1};
+use bdk_wallet::bitcoin::Psbt;
 
 use hwi::error::Error;
 use hwi::types::{HWIChain, HWIDevice};
 use hwi::HWIClient;
 
-use bdk::signer::{SignerCommon, SignerError, SignerId, TransactionSigner};
+use bdk_wallet::signer::{SignerCommon, SignerError, SignerId, TransactionSigner};
 
 #[derive(Debug)]
 /// Custom signer for Hardware Wallets
@@ -37,8 +37,8 @@ impl SignerCommon for HWISigner {
 impl TransactionSigner for HWISigner {
     fn sign_transaction(
         &self,
-        psbt: &mut PartiallySignedTransaction,
-        _sign_options: &bdk::SignOptions,
+        psbt: &mut Psbt,
+        _sign_options: &bdk_wallet::SignOptions,
         _secp: &Secp256k1<All>,
     ) -> Result<(), SignerError> {
         psbt.combine(
@@ -61,9 +61,9 @@ impl TransactionSigner for HWISigner {
 //     fn test_hardware_signer() {
 //         use std::sync::Arc;
 //
-//         use bdk::tests::get_funded_wallet;
-//         use bdk::signer::SignerOrdering;
-//         use bdk::bitcoin::Network;
+//         use bdk_wallet::tests::get_funded_wallet;
+//         use bdk_wallet::signer::SignerOrdering;
+//         use bdk_wallet::bitcoin::Network;
 //         use crate::HWISigner;
 //         use hwi::HWIClient;
 //
@@ -78,12 +78,12 @@ impl TransactionSigner for HWISigner {
 //
 //         let (mut wallet, _) = get_funded_wallet(&descriptors.internal[0]);
 //         wallet.add_signer(
-//             bdk::KeychainKind::External,
+//             bdk_wallet::KeychainKind::External,
 //             SignerOrdering(200),
 //             Arc::new(custom_signer),
 //         );
 //
-//         let addr = wallet.get_address(bdk::wallet::AddressIndex::LastUnused);
+//         let addr = wallet.get_address(bdk_wallet::AddressIndex::LastUnused);
 //         let mut builder = wallet.build_tx();
 //         builder.drain_to(addr.script_pubkey()).drain_wallet();
 //         let (mut psbt, _) = builder.finish().unwrap();

crates/testenv/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "bdk_testenv"
+version = "0.7.0"
+edition = "2021"
+rust-version = "1.63"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_testenv"
+description = "Testing framework for BDK chain sources."
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bdk_chain = { path = "../chain", version = "0.17", default-features = false }
+electrsd = { version = "0.28.0", features = ["bitcoind_25_0", "esplora_a33e97e1", "legacy"] }
+
+[features]
+default = ["std"]
+std = ["bdk_chain/std"]
+serde = ["bdk_chain/serde"]

crates/testenv/README.md

@@ -0,0 +1,6 @@
+# BDK TestEnv
+
+This crate sets up a regtest environment with a single [`bitcoind`] node
+connected to an [`electrs`] instance. This framework provides the infrastructure
+for testing chain source crates, e.g., [`bdk_chain`], [`bdk_electrum`],
+[`bdk_esplora`], etc.

crates/testenv/src/lib.rs

@@ -0,0 +1,304 @@
+use bdk_chain::{
+    bitcoin::{
+        address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash,
+        secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget,
+        ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid,
+    },
+    local_chain::CheckPoint,
+    BlockId,
+};
+use bitcoincore_rpc::{
+    bitcoincore_rpc_json::{GetBlockTemplateModes, GetBlockTemplateRules},
+    RpcApi,
+};
+pub use electrsd;
+pub use electrsd::bitcoind;
+pub use electrsd::bitcoind::anyhow;
+pub use electrsd::bitcoind::bitcoincore_rpc;
+pub use electrsd::electrum_client;
+use electrsd::electrum_client::ElectrumApi;
+use std::time::Duration;
+
+/// Struct for running a regtest environment with a single `bitcoind` node with an `electrs`
+/// instance connected to it.
+pub struct TestEnv {
+    pub bitcoind: electrsd::bitcoind::BitcoinD,
+    pub electrsd: electrsd::ElectrsD,
+}
+
+impl TestEnv {
+    /// Construct a new [`TestEnv`] instance with default configurations.
+    pub fn new() -> anyhow::Result<Self> {
+        let bitcoind = match std::env::var_os("BITCOIND_EXE") {
+            Some(bitcoind_path) => electrsd::bitcoind::BitcoinD::new(bitcoind_path),
+            None => {
+                let bitcoind_exe = electrsd::bitcoind::downloaded_exe_path()
+                    .expect(
+                "you need to provide an env var BITCOIND_EXE or specify a bitcoind version feature",
+                );
+                electrsd::bitcoind::BitcoinD::with_conf(
+                    bitcoind_exe,
+                    &electrsd::bitcoind::Conf::default(),
+                )
+            }
+        }?;
+
+        let mut electrsd_conf = electrsd::Conf::default();
+        electrsd_conf.http_enabled = true;
+        let electrsd = match std::env::var_os("ELECTRS_EXE") {
+            Some(env_electrs_exe) => {
+                electrsd::ElectrsD::with_conf(env_electrs_exe, &bitcoind, &electrsd_conf)
+            }
+            None => {
+                let electrs_exe = electrsd::downloaded_exe_path()
+                    .expect("electrs version feature must be enabled");
+                electrsd::ElectrsD::with_conf(electrs_exe, &bitcoind, &electrsd_conf)
+            }
+        }?;
+
+        Ok(Self { bitcoind, electrsd })
+    }
+
+    /// Exposes the [`ElectrumApi`] calls from the Electrum client.
+    pub fn electrum_client(&self) -> &impl ElectrumApi {
+        &self.electrsd.client
+    }
+
+    /// Exposes the [`RpcApi`] calls from [`bitcoincore_rpc`].
+    pub fn rpc_client(&self) -> &impl RpcApi {
+        &self.bitcoind.client
+    }
+
+    // Reset `electrsd` so that new blocks can be seen.
+    pub fn reset_electrsd(mut self) -> anyhow::Result<Self> {
+        let mut electrsd_conf = electrsd::Conf::default();
+        electrsd_conf.http_enabled = true;
+        let electrsd = match std::env::var_os("ELECTRS_EXE") {
+            Some(env_electrs_exe) => {
+                electrsd::ElectrsD::with_conf(env_electrs_exe, &self.bitcoind, &electrsd_conf)
+            }
+            None => {
+                let electrs_exe = electrsd::downloaded_exe_path()
+                    .expect("electrs version feature must be enabled");
+                electrsd::ElectrsD::with_conf(electrs_exe, &self.bitcoind, &electrsd_conf)
+            }
+        }?;
+        self.electrsd = electrsd;
+        Ok(self)
+    }
+
+    /// Mine a number of blocks of a given size `count`, which may be specified to a given coinbase
+    /// `address`.
+    pub fn mine_blocks(
+        &self,
+        count: usize,
+        address: Option<Address>,
+    ) -> anyhow::Result<Vec<BlockHash>> {
+        let coinbase_address = match address {
+            Some(address) => address,
+            None => self
+                .bitcoind
+                .client
+                .get_new_address(None, None)?
+                .assume_checked(),
+        };
+        let block_hashes = self
+            .bitcoind
+            .client
+            .generate_to_address(count as _, &coinbase_address)?;
+        Ok(block_hashes)
+    }
+
+    /// Mine a block that is guaranteed to be empty even with transactions in the mempool.
+    pub fn mine_empty_block(&self) -> anyhow::Result<(usize, BlockHash)> {
+        let bt = self.bitcoind.client.get_block_template(
+            GetBlockTemplateModes::Template,
+            &[GetBlockTemplateRules::SegWit],
+            &[],
+        )?;
+
+        let txdata = vec![Transaction {
+            version: transaction::Version::ONE,
+            lock_time: bdk_chain::bitcoin::absolute::LockTime::from_height(0)?,
+            input: vec![TxIn {
+                previous_output: bdk_chain::bitcoin::OutPoint::default(),
+                script_sig: ScriptBuf::builder()
+                    .push_int(bt.height as _)
+                    // randomn number so that re-mining creates unique block
+                    .push_int(random())
+                    .into_script(),
+                sequence: bdk_chain::bitcoin::Sequence::default(),
+                witness: bdk_chain::bitcoin::Witness::new(),
+            }],
+            output: vec![TxOut {
+                value: Amount::ZERO,
+                script_pubkey: ScriptBuf::new_p2sh(&ScriptHash::all_zeros()),
+            }],
+        }];
+
+        let bits: [u8; 4] = bt
+            .bits
+            .clone()
+            .try_into()
+            .expect("rpc provided us with invalid bits");
+
+        let mut block = Block {
+            header: Header {
+                version: bdk_chain::bitcoin::block::Version::default(),
+                prev_blockhash: bt.previous_block_hash,
+                merkle_root: TxMerkleNode::all_zeros(),
+                time: Ord::max(bt.min_time, std::time::UNIX_EPOCH.elapsed()?.as_secs()) as u32,
+                bits: CompactTarget::from_consensus(u32::from_be_bytes(bits)),
+                nonce: 0,
+            },
+            txdata,
+        };
+
+        block.header.merkle_root = block.compute_merkle_root().expect("must compute");
+
+        for nonce in 0..=u32::MAX {
+            block.header.nonce = nonce;
+            if block.header.target().is_met_by(block.block_hash()) {
+                break;
+            }
+        }
+
+        self.bitcoind.client.submit_block(&block)?;
+        Ok((bt.height as usize, block.block_hash()))
+    }
+
+    /// This method waits for the Electrum notification indicating that a new block has been mined.
+    pub fn wait_until_electrum_sees_block(&self) -> anyhow::Result<()> {
+        self.electrsd.client.block_headers_subscribe()?;
+        let mut delay = Duration::from_millis(64);
+
+        loop {
+            self.electrsd.trigger()?;
+            self.electrsd.client.ping()?;
+            if self.electrsd.client.block_headers_pop()?.is_some() {
+                return Ok(());
+            }
+
+            if delay.as_millis() < 512 {
+                delay = delay.mul_f32(2.0);
+            }
+            std::thread::sleep(delay);
+        }
+    }
+
+    /// Invalidate a number of blocks of a given size `count`.
+    pub fn invalidate_blocks(&self, count: usize) -> anyhow::Result<()> {
+        let mut hash = self.bitcoind.client.get_best_block_hash()?;
+        for _ in 0..count {
+            let prev_hash = self
+                .bitcoind
+                .client
+                .get_block_info(&hash)?
+                .previousblockhash;
+            self.bitcoind.client.invalidate_block(&hash)?;
+            match prev_hash {
+                Some(prev_hash) => hash = prev_hash,
+                None => break,
+            }
+        }
+        Ok(())
+    }
+
+    /// Reorg a number of blocks of a given size `count`.
+    /// Refer to [`TestEnv::mine_empty_block`] for more information.
+    pub fn reorg(&self, count: usize) -> anyhow::Result<Vec<BlockHash>> {
+        let start_height = self.bitcoind.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = self.mine_blocks(count, None);
+        assert_eq!(
+            self.bitcoind.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        res
+    }
+
+    /// Reorg with a number of empty blocks of a given size `count`.
+    pub fn reorg_empty_blocks(&self, count: usize) -> anyhow::Result<Vec<(usize, BlockHash)>> {
+        let start_height = self.bitcoind.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = (0..count)
+            .map(|_| self.mine_empty_block())
+            .collect::<Result<Vec<_>, _>>()?;
+        assert_eq!(
+            self.bitcoind.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        Ok(res)
+    }
+
+    /// Send a tx of a given `amount` to a given `address`.
+    pub fn send(&self, address: &Address<NetworkChecked>, amount: Amount) -> anyhow::Result<Txid> {
+        let txid = self
+            .bitcoind
+            .client
+            .send_to_address(address, amount, None, None, None, None, None, None)?;
+        Ok(txid)
+    }
+
+    /// Create a checkpoint linked list of all the blocks in the chain.
+    pub fn make_checkpoint_tip(&self) -> CheckPoint {
+        CheckPoint::from_block_ids((0_u32..).map_while(|height| {
+            self.bitcoind
+                .client
+                .get_block_hash(height as u64)
+                .ok()
+                .map(|hash| BlockId { height, hash })
+        }))
+        .expect("must craft tip")
+    }
+
+    /// Get the genesis hash of the blockchain.
+    pub fn genesis_hash(&self) -> anyhow::Result<BlockHash> {
+        let hash = self.bitcoind.client.get_block_hash(0)?;
+        Ok(hash)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::TestEnv;
+    use electrsd::bitcoind::{anyhow::Result, bitcoincore_rpc::RpcApi};
+
+    /// This checks that reorgs initiated by `bitcoind` is detected by our `electrsd` instance.
+    #[test]
+    fn test_reorg_is_detected_in_electrsd() -> Result<()> {
+        let env = TestEnv::new()?;
+
+        // Mine some blocks.
+        env.mine_blocks(101, None)?;
+        env.wait_until_electrum_sees_block()?;
+        let height = env.bitcoind.client.get_block_count()?;
+        let blocks = (0..=height)
+            .map(|i| env.bitcoind.client.get_block_hash(i))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        // Perform reorg on six blocks.
+        env.reorg(6)?;
+        env.wait_until_electrum_sees_block()?;
+        let reorged_height = env.bitcoind.client.get_block_count()?;
+        let reorged_blocks = (0..=height)
+            .map(|i| env.bitcoind.client.get_block_hash(i))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        assert_eq!(height, reorged_height);
+
+        // Block hashes should not be equal on the six reorged blocks.
+        for (i, (block, reorged_block)) in blocks.iter().zip(reorged_blocks.iter()).enumerate() {
+            match i <= height as usize - 6 {
+                true => assert_eq!(block, reorged_block),
+                false => assert_ne!(block, reorged_block),
+            }
+        }
+
+        Ok(())
+    }
+}

crates/wallet/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
-name = "bdk"
+name = "bdk_wallet"
 homepage = "https://bitcoindevkit.org"
-version = "1.0.0-alpha.6"
+version = "1.0.0-beta.1"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk"
 description = "A modern, lightweight, descriptor-based wallet library"
@@ -13,38 +13,35 @@ edition = "2021"
 rust-version = "1.63"
 
 [dependencies]
-rand = "^0.8"
-miniscript = { version = "10.0.0", features = ["serde"], default-features = false }
-bitcoin = { version = "0.30.0", features = ["serde", "base64", "rand-std"], default-features = false }
+rand_core = { version = "0.6.0" }
+miniscript = { version = "12.0.0", features = ["serde"], default-features = false }
+bitcoin = { version = "0.32.0", features = ["serde", "base64"], default-features = false }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { path = "../chain", version = "0.10.0", features = ["miniscript", "serde"], default-features = false }
+bdk_chain = { path = "../chain", version = "0.17.0", features = ["miniscript", "serde"], default-features = false }
+bdk_file_store = { path = "../file_store", version = "0.14.0", optional = true }
 
 # Optional dependencies
 bip39 = { version = "2.0", optional = true }
 
-[target.'cfg(target_arch = "wasm32")'.dependencies]
-getrandom = "0.2"
-js-sys = "0.3"
-
 [features]
 default = ["std"]
-std = ["bitcoin/std", "miniscript/std", "bdk_chain/std"]
+std = ["bitcoin/std", "bitcoin/rand-std", "miniscript/std", "bdk_chain/std"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]
-
-# This feature is used to run `cargo check` in our CI targeting wasm. It's not recommended
-# for libraries to explicitly include the "getrandom/js" feature, so we only do it when
-# necessary for running our CI. See: https://docs.rs/getrandom/0.2.8/getrandom/#webassembly-support
-dev-getrandom-wasm = ["getrandom/js"]
+rusqlite = ["bdk_chain/rusqlite"]
+file_store = ["bdk_file_store"]
 
 [dev-dependencies]
 lazy_static = "1.4"
 assert_matches = "1.5.0"
 tempfile = "3"
+bdk_chain = { path = "../chain", features = ["rusqlite"] }
+bdk_wallet = { path = ".", features = ["rusqlite", "file_store"] }
 bdk_file_store = { path = "../file_store" }
 anyhow = "1"
+rand = "^0.8"
 
 [package.metadata.docs.rs]
 all-features = true

crates/wallet/README.md

@@ -8,11 +8,11 @@
   </p>
 
   <p>
-    <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
+    <a href="https://crates.io/crates/bdk_wallet"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk_wallet.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
-    <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
+    <a href="https://docs.rs/bdk_wallet"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk_wallet-green"/></a>
     <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
@@ -20,13 +20,13 @@
   <h4>
     <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://docs.rs/bdk">Documentation</a>
+    <a href="https://docs.rs/bdk_wallet">Documentation</a>
   </h4>
 </div>
 
-## `bdk`
+# BDK Wallet
 
-The `bdk` crate provides the [`Wallet`](`crate::Wallet`) type which is a simple, high-level
+The `bdk_wallet` crate provides the [`Wallet`] type which is a simple, high-level
 interface built from the low-level components of [`bdk_chain`]. `Wallet` is a good starting point
 for many simple applications as well as a good demonstration of how to use the other mechanisms to
 construct a wallet. It has two keychains (external and internal) which are defined by
@@ -34,64 +34,80 @@ construct a wallet. It has two keychains (external and internal) which are defin
 chain data it also uses the descriptors to find transaction outputs owned by them. From there, you
 can create and sign transactions.
 
-For more information, see the [`Wallet`'s documentation](https://docs.rs/bdk/latest/bdk/wallet/struct.Wallet.html).
+For details about the API of `Wallet` see the [module-level documentation][`Wallet`].
 
-### Blockchain data
+## Blockchain data
 
-In order to get blockchain data for `Wallet` to consume, you have to put it into particular form.
-Right now this is [`KeychainScan`] which is defined in [`bdk_chain`].
-
-This can be created manually or from blockchain-scanning crates.
+In order to get blockchain data for `Wallet` to consume, you should configure a client from
+an available chain source. Typically you make a request to the chain source and get a response
+that the `Wallet` can use to update its view of the chain.
 
 **Blockchain Data Sources**
 
 * [`bdk_esplora`]: Grabs blockchain data from Esplora for updating BDK structures.
 * [`bdk_electrum`]: Grabs blockchain data from Electrum for updating BDK structures.
+* [`bdk_bitcoind_rpc`]: Grabs blockchain data from Bitcoin Core for updating BDK structures.
 
 **Examples**
 
-* [`example-crates/wallet_esplora`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora)
+* [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async)
+* [`example-crates/wallet_esplora_blocking`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_blocking)
 * [`example-crates/wallet_electrum`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_electrum)
+* [`example-crates/wallet_rpc`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_rpc)
 
-### Persistence
+## Persistence
 
-To persist the `Wallet` on disk, `Wallet` needs to be constructed with a
-[`Persist`](https://docs.rs/bdk_chain/latest/bdk_chain/keychain/struct.KeychainPersist.html) implementation.
+To persist `Wallet` state data use a data store crate that reads and writes [`ChangeSet`].
 
 **Implementations**
 
-* [`bdk_file_store`]: a simple flat-file implementation of `Persist`.
+* [`bdk_file_store`]: Stores wallet changes in a simple flat file.
 
 **Example**
 
-```rust
-use bdk::{bitcoin::Network, wallet::{AddressIndex, Wallet}};
+<!-- compile_fail because outpoint and txout are fake variables -->
+```rust,no_run
+use bdk_wallet::{bitcoin::Network, KeychainKind, ChangeSet, Wallet};
 
-fn main() {
-    // a type that implements `Persist`
-    let db = ();
+// Open or create a new file store for wallet data.
+let mut db =
+    bdk_file_store::Store::<ChangeSet>::open_or_create_new(b"magic_bytes", "/tmp/my_wallet.db")
+        .expect("create store");
 
-    let descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/0/*)";
-    let mut wallet = Wallet::new(descriptor, None, db, Network::Testnet).expect("should create");
+// Create a wallet with initial wallet data read from the file store.
+let network = Network::Testnet;
+let descriptor = "wpkh(tprv8ZgxMBicQKsPdcAqYBpzAFwU5yxBUo88ggoBqu1qPcHUfSbKK1sKMLmC7EAk438btHQrSdu3jGGQa6PA71nvH5nkDexhLteJqkM4dQmWF9g/84'/1'/0'/0/*)";
+let change_descriptor = "wpkh(tprv8ZgxMBicQKsPdcAqYBpzAFwU5yxBUo88ggoBqu1qPcHUfSbKK1sKMLmC7EAk438btHQrSdu3jGGQa6PA71nvH5nkDexhLteJqkM4dQmWF9g/84'/1'/0'/1/*)";
+let wallet_opt = Wallet::load()
+    .descriptors(descriptor, change_descriptor)
+    .network(network)
+    .load_wallet(&mut db)
+    .expect("wallet");
+let mut wallet = match wallet_opt {
+    Some(wallet) => wallet,
+    None => Wallet::create(descriptor, change_descriptor)
+        .network(network)
+        .create_wallet(&mut db)
+        .expect("wallet"),
+};
 
-    // get a new address (this increments revealed derivation index)
-    println!("revealed address: {}", wallet.get_address(AddressIndex::New));
-    println!("staged changes: {:?}", wallet.staged());
-    // persist changes
-    wallet.commit().expect("must save");
-}
+// Get a new address to receive bitcoin.
+let receive_address = wallet.reveal_next_address(KeychainKind::External);
+// Persist staged wallet data changes to the file store.
+wallet.persist(&mut db).expect("persist");
+println!("Your new receive address is: {}", receive_address.address);
 ```
 
 <!-- ### Sync the balance of a descriptor -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::Wallet; -->
-<!-- use bdk::blockchain::ElectrumBlockchain; -->
-<!-- use bdk::SyncOptions; -->
-<!-- use bdk::electrum_client::Client; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::Wallet; -->
+<!-- use bdk_wallet::blockchain::ElectrumBlockchain; -->
+<!-- use bdk_wallet::SyncOptions; -->
+<!-- use bdk_wallet::electrum_client::Client; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
 <!--     let blockchain = ElectrumBlockchain::from(Client::new("ssl://electrum.blockstream.info:60002")?); -->
 <!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
@@ -101,7 +117,7 @@ fn main() {
 
 <!--     wallet.sync(&blockchain, SyncOptions::default())?; -->
 
-<!--     println!("Descriptor balance: {} SAT", wallet.get_balance()?); -->
+<!--     println!("Descriptor balance: {} SAT", wallet.balance()?); -->
 
 <!--     Ok(()) -->
 <!-- } -->
@@ -109,12 +125,12 @@ fn main() {
 <!-- ### Generate a few addresses -->
 
 <!-- ```rust -->
-<!-- use bdk::Wallet; -->
-<!-- use bdk::wallet::AddressIndex::New; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::Wallet; -->
+<!-- use bdk_wallet::AddressIndex::New; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -131,19 +147,19 @@ fn main() {
 <!-- ### Create a transaction -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::{FeeRate, Wallet, SyncOptions}; -->
-<!-- use bdk::blockchain::ElectrumBlockchain; -->
+<!-- use bdk_wallet::{FeeRate, Wallet, SyncOptions}; -->
+<!-- use bdk_wallet::blockchain::ElectrumBlockchain; -->
 
-<!-- use bdk::electrum_client::Client; -->
-<!-- use bdk::wallet::AddressIndex::New; -->
+<!-- use bdk_wallet::electrum_client::Client; -->
+<!-- use bdk_wallet::AddressIndex::New; -->
 
 <!-- use bitcoin::base64; -->
-<!-- use bdk::bitcoin::consensus::serialize; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::bitcoin::consensus::serialize; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
 <!--     let blockchain = ElectrumBlockchain::from(Client::new("ssl://electrum.blockstream.info:60002")?); -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -172,14 +188,14 @@ fn main() {
 <!-- ### Sign a transaction -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::{Wallet, SignOptions}; -->
+<!-- use bdk_wallet::{Wallet, SignOptions}; -->
 
 <!-- use bitcoin::base64; -->
-<!-- use bdk::bitcoin::consensus::deserialize; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::bitcoin::consensus::deserialize; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -202,7 +218,7 @@ fn main() {
 cargo test
 ```
 
-## License
+# License
 
 Licensed under either of
 
@@ -211,16 +227,17 @@ Licensed under either of
 
 at your option.
 
-### Contribution
+# Contribution
 
 Unless you explicitly state otherwise, any contribution intentionally
 submitted for inclusion in the work by you, as defined in the Apache-2.0
 license, shall be dual licensed as above, without any additional terms or
 conditions.
 
+[`Wallet`]: https://docs.rs/bdk_wallet/latest/bdk_wallet/wallet/struct.Wallet.html
 [`bdk_chain`]: https://docs.rs/bdk_chain/latest
 [`bdk_file_store`]: https://docs.rs/bdk_file_store/latest
 [`bdk_electrum`]: https://docs.rs/bdk_electrum/latest
 [`bdk_esplora`]: https://docs.rs/bdk_esplora/latest
-[`KeychainScan`]: https://docs.rs/bdk_chain/latest/bdk_chain/keychain/struct.KeychainScan.html
+[`bdk_bitcoind_rpc`]: https://docs.rs/bdk_bitcoind_rpc/latest
 [`rust-miniscript`]: https://docs.rs/miniscript/latest/miniscript/index.html

crates/wallet/examples/compiler.rs

@@ -0,0 +1,98 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+extern crate bdk_wallet;
+extern crate bitcoin;
+extern crate miniscript;
+extern crate serde_json;
+
+use std::error::Error;
+use std::str::FromStr;
+
+use bitcoin::Network;
+use miniscript::policy::Concrete;
+use miniscript::Descriptor;
+
+use bdk_wallet::{KeychainKind, Wallet};
+
+/// Miniscript policy is a high level abstraction of spending conditions. Defined in the
+/// rust-miniscript library here  https://docs.rs/miniscript/7.0.0/miniscript/policy/index.html
+/// rust-miniscript provides a `compile()` function that can be used to compile any miniscript policy
+/// into a descriptor. This descriptor then in turn can be used in bdk a fully functioning wallet
+/// can be derived from the policy.
+///
+/// This example demonstrates the interaction between a bdk wallet and miniscript policy.
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // We start with a miniscript policy string
+    let policy_str = "or(
+        10@thresh(4,
+            pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2)
+        ),1@and(
+            older(4209713),
+            thresh(2,
+                pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),pk(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),pk(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068)
+            )
+        )
+    )"
+    .replace(&[' ', '\n', '\t'][..], "");
+
+    println!("Compiling policy: \n{}", policy_str);
+
+    // Parse the string as a [`Concrete`] type miniscript policy.
+    let policy = Concrete::<String>::from_str(&policy_str)?;
+
+    // Create a `wsh` type descriptor from the policy.
+    // `policy.compile()` returns the resulting miniscript from the policy.
+    let descriptor = Descriptor::new_wsh(policy.compile()?)?.to_string();
+
+    println!("Compiled into Descriptor: \n{}", descriptor);
+
+    // Do the same for another (internal) keychain
+    let policy_str = "or(
+        10@thresh(2,
+            pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec)
+        ),1@and(
+            pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),
+            older(12960)
+        )
+    )"
+    .replace(&[' ', '\n', '\t'][..], "");
+
+    println!("Compiling internal policy: \n{}", policy_str);
+
+    let policy = Concrete::<String>::from_str(&policy_str)?;
+    let internal_descriptor = Descriptor::new_wsh(policy.compile()?)?.to_string();
+    println!(
+        "Compiled into internal Descriptor: \n{}",
+        internal_descriptor
+    );
+
+    // Create a new wallet from descriptors
+    let mut wallet = Wallet::create(descriptor, internal_descriptor)
+        .network(Network::Regtest)
+        .create_wallet_no_persist()?;
+
+    println!(
+        "First derived address from the descriptor: \n{}",
+        wallet.next_unused_address(KeychainKind::External),
+    );
+
+    // BDK also has it's own `Policy` structure to represent the spending condition in a more
+    // human readable json format.
+    let spending_policy = wallet.policies(KeychainKind::External)?;
+    println!(
+        "The BDK spending policy: \n{}",
+        serde_json::to_string_pretty(&spending_policy)?
+    );
+
+    Ok(())
+}

crates/wallet/examples/mnemonic_to_descriptors.rs

@@ -7,14 +7,14 @@
 // licenses.
 
 use anyhow::anyhow;
-use bdk::bitcoin::bip32::DerivationPath;
-use bdk::bitcoin::secp256k1::Secp256k1;
-use bdk::bitcoin::Network;
-use bdk::descriptor;
-use bdk::descriptor::IntoWalletDescriptor;
-use bdk::keys::bip39::{Language, Mnemonic, WordCount};
-use bdk::keys::{GeneratableKey, GeneratedKey};
-use bdk::miniscript::Tap;
+use bdk_wallet::bitcoin::bip32::DerivationPath;
+use bdk_wallet::bitcoin::secp256k1::Secp256k1;
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::descriptor;
+use bdk_wallet::descriptor::IntoWalletDescriptor;
+use bdk_wallet::keys::bip39::{Language, Mnemonic, WordCount};
+use bdk_wallet::keys::{GeneratableKey, GeneratedKey};
+use bdk_wallet::miniscript::Tap;
 use std::str::FromStr;
 
 /// This example demonstrates how to generate a mnemonic phrase

crates/wallet/examples/policy.rs

@@ -9,14 +9,14 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-extern crate bdk;
+extern crate bdk_wallet;
 use std::error::Error;
 
-use bdk::bitcoin::Network;
-use bdk::descriptor::{policy::BuildSatisfaction, ExtractPolicy, IntoWalletDescriptor};
-use bdk::wallet::signer::SignersContainer;
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::descriptor::{policy::BuildSatisfaction, ExtractPolicy, IntoWalletDescriptor};
+use bdk_wallet::signer::SignersContainer;
 
-/// This example describes the use of the BDK's [`bdk::descriptor::policy`] module.
+/// This example describes the use of the BDK's [`bdk_wallet::descriptor::policy`] module.
 ///
 /// Policy is higher abstraction representation of the wallet descriptor spending condition.
 /// This is useful to express complex miniscript spending conditions into more human readable form.
@@ -34,11 +34,11 @@ fn main() -> Result<(), Box<dyn Error>> {
     let desc = "wsh(multi(2,tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/*))";
 
     // Use the descriptor string to derive the full descriptor and a keymap.
-    // The wallet descriptor can be used to create a new bdk::wallet.
+    // The wallet descriptor can be used to create a new bdk_wallet::wallet.
     // While the `keymap` can be used to create a `SignerContainer`.
     //
     // The `SignerContainer` can sign for `PSBT`s.
-    // a bdk::wallet internally uses these to handle transaction signing.
+    // a `bdk_wallet::Wallet` internally uses these to handle transaction signing.
     // But they can be used as independent tools also.
     let (wallet_desc, keymap) = desc.into_wallet_descriptor(&secp, Network::Testnet)?;
 

crates/wallet/src/descriptor/dsl.rs

@@ -274,14 +274,13 @@ macro_rules! impl_sortedmulti {
 #[macro_export]
 macro_rules! parse_tap_tree {
     ( @merge $tree_a:expr, $tree_b:expr) => {{
-        use $crate::alloc::sync::Arc;
         use $crate::miniscript::descriptor::TapTree;
 
         $tree_a
             .and_then(|tree_a| Ok((tree_a, $tree_b?)))
             .and_then(|((a_tree, mut a_keymap, a_networks), (b_tree, b_keymap, b_networks))| {
                 a_keymap.extend(b_keymap.into_iter());
-                Ok((TapTree::Tree(Arc::new(a_tree), Arc::new(b_tree)), a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
+                Ok((TapTree::combine(a_tree, b_tree), a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
             })
 
     }};
@@ -424,7 +423,7 @@ macro_rules! apply_modifier {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// let (my_descriptor, my_keys_map, networks) = bdk::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
+/// let (my_descriptor, my_keys_map, networks) = bdk_wallet::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
 /// # Ok::<(), Box<dyn std::error::Error>>(())
 /// ```
 ///
@@ -445,7 +444,7 @@ macro_rules! apply_modifier {
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 /// let my_timelock = 50;
 ///
-/// let (descriptor_a, key_map_a, networks) = bdk::descriptor! {
+/// let (descriptor_a, key_map_a, networks) = bdk_wallet::descriptor! {
 ///     wsh (
 ///         thresh(2, pk(my_key_1), s:pk(my_key_2), s:n:d:v:older(my_timelock))
 ///     )
@@ -453,11 +452,12 @@ macro_rules! apply_modifier {
 ///
 /// #[rustfmt::skip]
 /// let b_items = vec![
-///     bdk::fragment!(pk(my_key_1))?,
-///     bdk::fragment!(s:pk(my_key_2))?,
-///     bdk::fragment!(s:n:d:v:older(my_timelock))?,
+///     bdk_wallet::fragment!(pk(my_key_1))?,
+///     bdk_wallet::fragment!(s:pk(my_key_2))?,
+///     bdk_wallet::fragment!(s:n:d:v:older(my_timelock))?,
 /// ];
-/// let (descriptor_b, mut key_map_b, networks) = bdk::descriptor!(wsh(thresh_vec(2, b_items)))?;
+/// let (descriptor_b, mut key_map_b, networks) =
+///     bdk_wallet::descriptor!(wsh(thresh_vec(2, b_items)))?;
 ///
 /// assert_eq!(descriptor_a, descriptor_b);
 /// assert_eq!(key_map_a.len(), key_map_b.len());
@@ -476,7 +476,7 @@ macro_rules! apply_modifier {
 /// let my_key_2 =
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 ///
-/// let (descriptor, key_map, networks) = bdk::descriptor! {
+/// let (descriptor, key_map, networks) = bdk_wallet::descriptor! {
 ///     wsh (
 ///         multi(2, my_key_1, my_key_2)
 ///     )
@@ -492,7 +492,7 @@ macro_rules! apply_modifier {
 /// let my_key =
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 ///
-/// let (descriptor, key_map, networks) = bdk::descriptor!(wpkh(my_key))?;
+/// let (descriptor, key_map, networks) = bdk_wallet::descriptor!(wpkh(my_key))?;
 /// # Ok::<(), Box<dyn std::error::Error>>(())
 /// ```
 ///
@@ -703,10 +703,10 @@ macro_rules! fragment {
         $crate::keys::make_pkh($key, &secp)
     });
     ( after ( $value:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(After, $crate::miniscript::AbsLockTime::from_consensus($value))
+        $crate::impl_leaf_opcode_value!(After, $crate::miniscript::AbsLockTime::from_consensus($value).expect("valid `AbsLockTime`"))
     });
     ( older ( $value:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(Older, $crate::bitcoin::Sequence($value)) // TODO!!
+        $crate::impl_leaf_opcode_value!(Older, $crate::miniscript::RelLockTime::from_consensus($value).expect("valid `RelLockTime`")) // TODO!!
     });
     ( sha256 ( $hash:expr ) ) => ({
         $crate::impl_leaf_opcode_value!(Sha256, $hash)
@@ -757,7 +757,8 @@ macro_rules! fragment {
             (keys_acc, net_acc)
         });
 
-        $crate::impl_leaf_opcode_value_two!(Thresh, $thresh, items)
+        let thresh = $crate::miniscript::Threshold::new($thresh, items).expect("valid threshold and pks collection");
+        $crate::impl_leaf_opcode_value!(Thresh, thresh)
             .map(|(minisc, _, _)| (minisc, key_maps, valid_networks))
     });
     ( thresh ( $thresh:expr, $( $inner:tt )* ) ) => ({
@@ -769,7 +770,12 @@ macro_rules! fragment {
     ( multi_vec ( $thresh:expr, $keys:expr ) ) => ({
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        $crate::keys::make_multi($thresh, $crate::miniscript::Terminal::Multi, $keys, &secp)
+        let fun = |k, pks| {
+            let thresh = $crate::miniscript::Threshold::new(k, pks).expect("valid threshold and pks collection");
+            $crate::miniscript::Terminal::Multi(thresh)
+        };
+
+        $crate::keys::make_multi($thresh, fun, $keys, &secp)
     });
     ( multi ( $thresh:expr $(, $key:expr )+ ) ) => ({
         $crate::group_multi_keys!( $( $key ),* )
@@ -778,7 +784,12 @@ macro_rules! fragment {
     ( multi_a_vec ( $thresh:expr, $keys:expr ) ) => ({
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        $crate::keys::make_multi($thresh, $crate::miniscript::Terminal::MultiA, $keys, &secp)
+        let fun = |k, pks| {
+            let thresh = $crate::miniscript::Threshold::new(k, pks).expect("valid threshold and pks collection");
+            $crate::miniscript::Terminal::MultiA(thresh)
+        };
+
+        $crate::keys::make_multi($thresh, fun, $keys, &secp)
     });
     ( multi_a ( $thresh:expr $(, $key:expr )+ ) ) => ({
         $crate::group_multi_keys!( $( $key ),* )
@@ -806,7 +817,7 @@ mod test {
     use crate::descriptor::{DescriptorError, DescriptorMeta};
     use crate::keys::{DescriptorKey, IntoDescriptorKey, ValidNetworks};
     use bitcoin::bip32;
-    use bitcoin::network::constants::Network::{Bitcoin, Regtest, Signet, Testnet};
+    use bitcoin::Network::{Bitcoin, Regtest, Signet, Testnet};
     use bitcoin::PrivateKey;
 
     // test the descriptor!() macro
@@ -936,7 +947,7 @@ mod test {
 
     #[test]
     fn test_bip32_legacy_descriptors() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
 
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
@@ -981,7 +992,7 @@ mod test {
 
     #[test]
     fn test_bip32_segwitv0_descriptors() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
 
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
@@ -1038,10 +1049,10 @@ mod test {
 
     #[test]
     fn test_dsl_sortedmulti() {
-        let key_1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let key_1 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path_1 = bip32::DerivationPath::from_str("m/0").unwrap();
 
-        let key_2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let key_2 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
         let path_2 = bip32::DerivationPath::from_str("m/1").unwrap();
 
         let desc_key1 = (key_1, path_1);
@@ -1097,7 +1108,7 @@ mod test {
     // - verify the valid_networks returned is correctly computed based on the keys present in the descriptor
     #[test]
     fn test_valid_networks() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
@@ -1107,7 +1118,7 @@ mod test {
             [Testnet, Regtest, Signet].iter().cloned().collect()
         );
 
-        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
+        let xprv = bip32::Xpriv::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
         let path = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
         let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
@@ -1120,15 +1131,15 @@ mod test {
     fn test_key_maps_merged() {
         let secp = Secp256k1::new();
 
-        let xprv1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv1 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path1 = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key1 = (xprv1, path1.clone()).into_descriptor_key().unwrap();
 
-        let xprv2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let xprv2 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
         let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
         let desc_key2 = (xprv2, path2.clone()).into_descriptor_key().unwrap();
 
-        let xprv3 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
+        let xprv3 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
         let path3 = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
         let desc_key3 = (xprv3, path3.clone()).into_descriptor_key().unwrap();
 
@@ -1152,7 +1163,7 @@ mod test {
     #[test]
     fn test_script_context_validation() {
         // this compiles
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key: DescriptorKey<Legacy> = (xprv, path).into_descriptor_key().unwrap();
 

crates/wallet/src/descriptor/error.rs

@@ -13,7 +13,7 @@
 use core::fmt;
 
 /// Errors related to the parsing and usage of descriptors
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum Error {
     /// Invalid HD Key path, such as having a wildcard but a length != 1
     InvalidHdKeyPath,
@@ -23,7 +23,6 @@ pub enum Error {
     HardenedDerivationXpub,
     /// The descriptor contains multipath keys
     MultiPath,
-
     /// Error thrown while working with [`keys`](crate::keys)
     Key(crate::keys::KeyError),
     /// Error while extracting and manipulating policies
@@ -37,11 +36,13 @@ pub enum Error {
     /// Error during base58 decoding
     Base58(bitcoin::base58::Error),
     /// Key-related error
-    Pk(bitcoin::key::Error),
+    Pk(bitcoin::key::ParsePublicKeyError),
     /// Miniscript error
     Miniscript(miniscript::Error),
     /// Hex decoding error
-    Hex(bitcoin::hashes::hex::Error),
+    Hex(bitcoin::hex::HexToBytesError),
+    /// The provided wallet descriptors are identical
+    ExternalAndInternalAreTheSame,
 }
 
 impl From<crate::keys::KeyError> for Error {
@@ -79,6 +80,9 @@ impl fmt::Display for Error {
             Self::Pk(err) => write!(f, "Key-related error: {}", err),
             Self::Miniscript(err) => write!(f, "Miniscript error: {}", err),
             Self::Hex(err) => write!(f, "Hex decoding error: {}", err),
+            Self::ExternalAndInternalAreTheSame => {
+                write!(f, "External and internal descriptors are the same")
+            }
         }
     }
 }
@@ -98,8 +102,8 @@ impl From<bitcoin::base58::Error> for Error {
     }
 }
 
-impl From<bitcoin::key::Error> for Error {
-    fn from(err: bitcoin::key::Error) -> Self {
+impl From<bitcoin::key::ParsePublicKeyError> for Error {
+    fn from(err: bitcoin::key::ParsePublicKeyError) -> Self {
         Error::Pk(err)
     }
 }
@@ -110,8 +114,8 @@ impl From<miniscript::Error> for Error {
     }
 }
 
-impl From<bitcoin::hashes::hex::Error> for Error {
-    fn from(err: bitcoin::hashes::hex::Error) -> Self {
+impl From<bitcoin::hex::HexToBytesError> for Error {
+    fn from(err: bitcoin::hex::HexToBytesError) -> Self {
         Error::Hex(err)
     }
 }

crates/wallet/src/descriptor/mod.rs

@@ -18,7 +18,7 @@ use crate::collections::BTreeMap;
 use alloc::string::String;
 use alloc::vec::Vec;
 
-use bitcoin::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource};
+use bitcoin::bip32::{ChildNumber, DerivationPath, Fingerprint, KeySource, Xpub};
 use bitcoin::{key::XOnlyPublicKey, secp256k1, PublicKey};
 use bitcoin::{psbt, taproot};
 use bitcoin::{Network, TxOut};
@@ -112,6 +112,16 @@ impl IntoWalletDescriptor for &String {
     }
 }
 
+impl IntoWalletDescriptor for String {
+    fn into_wallet_descriptor(
+        self,
+        secp: &SecpCtx,
+        network: Network,
+    ) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
+        self.as_str().into_wallet_descriptor(secp, network)
+    }
+}
+
 impl IntoWalletDescriptor for ExtendedDescriptor {
     fn into_wallet_descriptor(
         self,
@@ -229,7 +239,7 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
                 let pk = match pk {
                     DescriptorPublicKey::XPub(ref xpub) => {
                         let mut xpub = xpub.clone();
-                        xpub.xkey.network = self.network;
+                        xpub.xkey.network = self.network.into();
 
                         DescriptorPublicKey::XPub(xpub)
                     }
@@ -264,11 +274,11 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
             .map(|(mut k, mut v)| {
                 match (&mut k, &mut v) {
                     (DescriptorPublicKey::XPub(xpub), DescriptorSecretKey::XPrv(xprv)) => {
-                        xpub.xkey.network = network;
-                        xprv.xkey.network = network;
+                        xpub.xkey.network = network.into();
+                        xprv.xkey.network = network.into();
                     }
                     (_, DescriptorSecretKey::Single(key)) => {
-                        key.key.network = network;
+                        key.key.network = network.into();
                     }
                     _ => {}
                 }
@@ -281,15 +291,10 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
     }
 }
 
-/// Wrapper for `IntoWalletDescriptor` that performs additional checks on the keys contained in the
-/// descriptor
-pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
-    inner: T,
-    secp: &SecpCtx,
-    network: Network,
-) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError> {
-    let (descriptor, keymap) = inner.into_wallet_descriptor(secp, network)?;
-
+/// Extra checks for [`ExtendedDescriptor`].
+pub(crate) fn check_wallet_descriptor(
+    descriptor: &Descriptor<DescriptorPublicKey>,
+) -> Result<(), DescriptorError> {
     // Ensure the keys don't contain any hardened derivation steps or hardened wildcards
     let descriptor_contains_hardened_steps = descriptor.for_any_key(|k| {
         if let DescriptorPublicKey::XPub(DescriptorXKey {
@@ -316,7 +321,7 @@ pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
     // issues
     descriptor.sanity_check()?;
 
-    Ok((descriptor, keymap))
+    Ok(())
 }
 
 #[doc(hidden)]
@@ -377,7 +382,7 @@ where
 pub(crate) trait DescriptorMeta {
     fn is_witness(&self) -> bool;
     fn is_taproot(&self) -> bool;
-    fn get_extended_keys(&self) -> Vec<DescriptorXKey<ExtendedPubKey>>;
+    fn get_extended_keys(&self) -> Vec<DescriptorXKey<Xpub>>;
     fn derive_from_hd_keypaths(
         &self,
         hd_keypaths: &HdKeyPaths,
@@ -418,7 +423,7 @@ impl DescriptorMeta for ExtendedDescriptor {
         self.desc_type() == DescriptorType::Tr
     }
 
-    fn get_extended_keys(&self) -> Vec<DescriptorXKey<ExtendedPubKey>> {
+    fn get_extended_keys(&self) -> Vec<DescriptorXKey<Xpub>> {
         let mut answer = Vec::new();
 
         self.for_each_key(|pk| {
@@ -438,21 +443,20 @@ impl DescriptorMeta for ExtendedDescriptor {
         secp: &SecpCtx,
     ) -> Option<DerivedDescriptor> {
         // Ensure that deriving `xpub` with `path` yields `expected`
-        let verify_key = |xpub: &DescriptorXKey<ExtendedPubKey>,
-                          path: &DerivationPath,
-                          expected: &SinglePubKey| {
-            let derived = xpub
-                .xkey
-                .derive_pub(secp, path)
-                .expect("The path should never contain hardened derivation steps")
-                .public_key;
+        let verify_key =
+            |xpub: &DescriptorXKey<Xpub>, path: &DerivationPath, expected: &SinglePubKey| {
+                let derived = xpub
+                    .xkey
+                    .derive_pub(secp, path)
+                    .expect("The path should never contain hardened derivation steps")
+                    .public_key;
 
-            match expected {
-                SinglePubKey::FullKey(pk) if &PublicKey::new(derived) == pk => true,
-                SinglePubKey::XOnly(pk) if &XOnlyPublicKey::from(derived) == pk => true,
-                _ => false,
-            }
-        };
+                match expected {
+                    SinglePubKey::FullKey(pk) if &PublicKey::new(derived) == pk => true,
+                    SinglePubKey::XOnly(pk) if &XOnlyPublicKey::from(derived) == pk => true,
+                    _ => false,
+                }
+            };
 
         let mut path_found = None;
 
@@ -605,10 +609,10 @@ mod test {
     use core::str::FromStr;
 
     use assert_matches::assert_matches;
-    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::hex::FromHex;
     use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::ScriptBuf;
-    use bitcoin::{bip32, psbt::Psbt};
+    use bitcoin::{bip32, Psbt};
+    use bitcoin::{NetworkKind, ScriptBuf};
 
     use super::*;
     use crate::psbt::PsbtUtils;
@@ -727,7 +731,7 @@ mod test {
 
         let secp = Secp256k1::new();
 
-        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3c3gF1DUWpWNr2SG2XrG8oYPpqYh7hoWsJy9NjabErnzriJPpnGHyKz5NgdXmq1KVbqS1r4NXdCoKitWg5e86zqXHa8kxyB").unwrap();
+        let xprv = bip32::Xpriv::from_str("xprv9s21ZrQH143K3c3gF1DUWpWNr2SG2XrG8oYPpqYh7hoWsJy9NjabErnzriJPpnGHyKz5NgdXmq1KVbqS1r4NXdCoKitWg5e86zqXHa8kxyB").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
 
         // here `to_descriptor_key` will set the valid networks for the key to only mainnet, since
@@ -744,9 +748,9 @@ mod test {
             .unwrap();
 
         let mut xprv_testnet = xprv;
-        xprv_testnet.network = Network::Testnet;
+        xprv_testnet.network = NetworkKind::Test;
 
-        let xpub_testnet = bip32::ExtendedPubKey::from_priv(&secp, &xprv_testnet);
+        let xpub_testnet = bip32::Xpub::from_priv(&secp, &xprv_testnet);
         let desc_pubkey = DescriptorPublicKey::XPub(DescriptorXKey {
             xkey: xpub_testnet,
             origin: None,
@@ -836,7 +840,7 @@ mod test {
     fn test_descriptor_from_str_from_output_of_macro() {
         let secp = Secp256k1::new();
 
-        let tpub = bip32::ExtendedPubKey::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
+        let tpub = bip32::Xpub::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
         let path = bip32::DerivationPath::from_str("m/1/2").unwrap();
         let key = (tpub, path).into_descriptor_key().unwrap();
 
@@ -856,22 +860,31 @@ mod test {
     }
 
     #[test]
-    fn test_into_wallet_descriptor_checked() {
+    fn test_check_wallet_descriptor() {
         let secp = Secp256k1::new();
 
         let descriptor = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0'/1/2/*)";
-        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+        let (descriptor, _) = descriptor
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .expect("must parse");
+        let result = check_wallet_descriptor(&descriptor);
 
         assert_matches!(result, Err(DescriptorError::HardenedDerivationXpub));
 
         let descriptor = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/<0;1>/*)";
-        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+        let (descriptor, _) = descriptor
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .expect("must parse");
+        let result = check_wallet_descriptor(&descriptor);
 
         assert_matches!(result, Err(DescriptorError::MultiPath));
 
         // repeated pubkeys
         let descriptor = "wsh(multi(2,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*))";
-        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+        let (descriptor, _) = descriptor
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .expect("must parse");
+        let result = check_wallet_descriptor(&descriptor);
 
         assert!(result.is_err());
     }
@@ -883,8 +896,10 @@ mod test {
         let secp = Secp256k1::new();
 
         let descriptor = "sh(wsh(sortedmulti(3,tpubDEsqS36T4DVsKJd9UH8pAKzrkGBYPLEt9jZMwpKtzh1G6mgYehfHt9WCgk7MJG5QGSFWf176KaBNoXbcuFcuadAFKxDpUdMDKGBha7bY3QM/0/*,tpubDF3cpwfs7fMvXXuoQbohXtLjNM6ehwYT287LWtmLsd4r77YLg6MZg4vTETx5MSJ2zkfigbYWu31VA2Z2Vc1cZugCYXgS7FQu6pE8V6TriEH/0/*,tpubDE1SKfcW76Tb2AASv5bQWMuScYNAdoqLHoexw13sNDXwmUhQDBbCD3QAedKGLhxMrWQdMDKENzYtnXPDRvexQPNuDrLj52wAjHhNEm8sJ4p/0/*,tpubDFLc6oXwJmhm3FGGzXkfJNTh2KitoY3WhmmQvuAjMhD8YbyWn5mAqckbxXfm2etM3p5J6JoTpSrMqRSTfMLtNW46poDaEZJ1kjd3csRSjwH/0/*,tpubDEWD9NBeWP59xXmdqSNt4VYdtTGwbpyP8WS962BuqpQeMZmX9Pur14dhXdZT5a7wR1pK6dPtZ9fP5WR493hPzemnBvkfLLYxnUjAKj1JCQV/0/*,tpubDEHyZkkwd7gZWCTgQuYQ9C4myF2hMEmyHsBCCmLssGqoqUxeT3gzohF5uEVURkf9TtmeepJgkSUmteac38FwZqirjApzNX59XSHLcwaTZCH/0/*,tpubDEqLouCekwnMUWN486kxGzD44qVgeyuqHyxUypNEiQt5RnUZNJe386TKPK99fqRV1vRkZjYAjtXGTECz98MCsdLcnkM67U6KdYRzVubeCgZ/0/*)))";
-        let (descriptor, _) =
-            into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet).unwrap();
+        let (descriptor, _) = descriptor
+            .into_wallet_descriptor(&secp, Network::Testnet)
+            .unwrap();
+        check_wallet_descriptor(&descriptor).expect("descriptor");
 
         let descriptor = descriptor.at_derivation_index(0).unwrap();
 
@@ -895,7 +910,7 @@ mod test {
             .update_with_descriptor_unchecked(&descriptor)
             .unwrap();
 
-        assert_eq!(psbt_input.redeem_script, Some(script.to_v0_p2wsh()));
+        assert_eq!(psbt_input.redeem_script, Some(script.to_p2wsh()));
         assert_eq!(psbt_input.witness_script, Some(script));
     }
 }

crates/wallet/src/descriptor/policy.rs

@@ -20,10 +20,10 @@
 //!
 //! ```
 //! # use std::sync::Arc;
-//! # use bdk::descriptor::*;
-//! # use bdk::wallet::signer::*;
-//! # use bdk::bitcoin::secp256k1::Secp256k1;
-//! use bdk::descriptor::policy::BuildSatisfaction;
+//! # use bdk_wallet::descriptor::*;
+//! # use bdk_wallet::signer::*;
+//! # use bdk_wallet::bitcoin::secp256k1::Secp256k1;
+//! use bdk_wallet::descriptor::policy::BuildSatisfaction;
 //! let secp = Secp256k1::new();
 //! let desc = "wsh(and_v(v:pk(cV3oCth6zxZ1UVsHLnGothsWNsaoxRhC6aeNi5VbSdFpwUkgkEci),or_d(pk(cVMTy7uebJgvFaSBwcgvwk8qn8xSLc97dKow4MBetjrrahZoimm2),older(12960))))";
 //!
@@ -40,6 +40,7 @@ use crate::collections::{BTreeMap, HashSet, VecDeque};
 use alloc::string::String;
 use alloc::vec::Vec;
 use core::cmp::max;
+use miniscript::miniscript::limits::{MAX_PUBKEYS_IN_CHECKSIGADD, MAX_PUBKEYS_PER_MULTISIG};
 
 use core::fmt;
 
@@ -48,12 +49,12 @@ use serde::{Serialize, Serializer};
 
 use bitcoin::bip32::Fingerprint;
 use bitcoin::hashes::{hash160, ripemd160, sha256};
-use bitcoin::{absolute, key::XOnlyPublicKey, PublicKey, Sequence};
+use bitcoin::{absolute, key::XOnlyPublicKey, relative, PublicKey, Sequence};
 
 use miniscript::descriptor::{
     DescriptorPublicKey, ShInner, SinglePub, SinglePubKey, SortedMultiVec, WshInner,
 };
-use miniscript::hash256;
+use miniscript::{hash256, Threshold};
 use miniscript::{
     Descriptor, Miniscript, Satisfier, ScriptContext, SigType, Terminal, ToPublicKey,
 };
@@ -137,7 +138,7 @@ pub enum SatisfiableItem {
     /// Relative timelock locktime
     RelativeTimelock {
         /// The timelock value
-        value: Sequence,
+        value: relative::LockTime,
     },
     /// Multi-signature public keys with threshold count
     Multisig {
@@ -586,30 +587,25 @@ impl Policy {
         Ok(Some(policy))
     }
 
-    fn make_multisig<Ctx: ScriptContext + 'static>(
-        keys: &[DescriptorPublicKey],
+    fn make_multi<Ctx: ScriptContext + 'static, const MAX: usize>(
+        threshold: &Threshold<DescriptorPublicKey, MAX>,
         signers: &SignersContainer,
         build_sat: BuildSatisfaction,
-        threshold: usize,
         sorted: bool,
         secp: &SecpCtx,
     ) -> Result<Option<Policy>, PolicyError> {
-        if threshold == 0 {
-            return Ok(None);
-        }
-
-        let parsed_keys = keys.iter().map(|k| PkOrF::from_key(k, secp)).collect();
+        let parsed_keys = threshold.iter().map(|k| PkOrF::from_key(k, secp)).collect();
 
         let mut contribution = Satisfaction::Partial {
-            n: keys.len(),
-            m: threshold,
+            n: threshold.n(),
+            m: threshold.k(),
             items: vec![],
             conditions: Default::default(),
             sorted: Some(sorted),
         };
         let mut satisfaction = contribution.clone();
 
-        for (index, key) in keys.iter().enumerate() {
+        for (index, key) in threshold.iter().enumerate() {
             if signers.find(signer_id(key, secp)).is_some() {
                 contribution.add(
                     &Satisfaction::Complete {
@@ -618,7 +614,6 @@ impl Policy {
                     index,
                 )?;
             }
-
             if let Some(psbt) = build_sat.psbt() {
                 if Ctx::find_signature(psbt, key, secp) {
                     satisfaction.add(
@@ -635,12 +630,11 @@ impl Policy {
 
         let mut policy: Policy = SatisfiableItem::Multisig {
             keys: parsed_keys,
-            threshold,
+            threshold: threshold.k(),
         }
         .into();
         policy.contribution = contribution;
         policy.satisfaction = satisfaction;
-
         Ok(Some(policy))
     }
 
@@ -725,7 +719,7 @@ impl Policy {
                 timelock: Some(*value),
             }),
             SatisfiableItem::RelativeTimelock { value } => Ok(Condition {
-                csv: Some(*value),
+                csv: Some((*value).into()),
                 timelock: None,
             }),
             _ => Ok(Condition::default()),
@@ -952,11 +946,14 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 Some(policy)
             }
             Terminal::Older(value) => {
-                let mut policy: Policy = SatisfiableItem::RelativeTimelock { value: *value }.into();
+                let mut policy: Policy = SatisfiableItem::RelativeTimelock {
+                    value: (*value).into(),
+                }
+                .into();
                 policy.contribution = Satisfaction::Complete {
                     condition: Condition {
                         timelock: None,
-                        csv: Some(*value),
+                        csv: Some((*value).into()),
                     },
                 };
                 if let BuildSatisfaction::PsbtTimelocks {
@@ -966,9 +963,11 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 } = build_sat
                 {
                     let older = Older::new(Some(current_height), Some(input_max_height), false);
-                    let older_sat = Satisfier::<bitcoin::PublicKey>::check_older(&older, *value);
-                    let inputs_sat = psbt_inputs_sat(psbt)
-                        .all(|sat| Satisfier::<bitcoin::PublicKey>::check_older(&sat, *value));
+                    let older_sat =
+                        Satisfier::<bitcoin::PublicKey>::check_older(&older, (*value).into());
+                    let inputs_sat = psbt_inputs_sat(psbt).all(|sat| {
+                        Satisfier::<bitcoin::PublicKey>::check_older(&sat, (*value).into())
+                    });
                     if older_sat && inputs_sat {
                         policy.satisfaction = policy.contribution.clone();
                     }
@@ -986,9 +985,12 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
             Terminal::Hash160(hash) => {
                 Some(SatisfiableItem::Hash160Preimage { hash: *hash }.into())
             }
-            Terminal::Multi(k, pks) | Terminal::MultiA(k, pks) => {
-                Policy::make_multisig::<Ctx>(pks, signers, build_sat, *k, false, secp)?
-            }
+            Terminal::Multi(threshold) => Policy::make_multi::<Ctx, MAX_PUBKEYS_PER_MULTISIG>(
+                threshold, signers, build_sat, false, secp,
+            )?,
+            Terminal::MultiA(threshold) => Policy::make_multi::<Ctx, MAX_PUBKEYS_IN_CHECKSIGADD>(
+                threshold, signers, build_sat, false, secp,
+            )?,
             // Identities
             Terminal::Alt(inner)
             | Terminal::Swap(inner)
@@ -1016,8 +1018,9 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 a.extract_policy(signers, build_sat, secp)?,
                 b.extract_policy(signers, build_sat, secp)?,
             )?,
-            Terminal::Thresh(k, nodes) => {
-                let mut threshold = *k;
+            Terminal::Thresh(threshold) => {
+                let mut k = threshold.k();
+                let nodes = threshold.data();
                 let mapped: Vec<_> = nodes
                     .iter()
                     .map(|n| n.extract_policy(signers, build_sat, secp))
@@ -1027,13 +1030,13 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                     .collect();
 
                 if mapped.len() < nodes.len() {
-                    threshold = match threshold.checked_sub(nodes.len() - mapped.len()) {
+                    k = match k.checked_sub(nodes.len() - mapped.len()) {
                         None => return Ok(None),
                         Some(x) => x,
                     };
                 }
 
-                Policy::make_thresh(mapped, threshold)?
+                Policy::make_thresh(mapped, k)?
             }
 
             // Unsupported
@@ -1087,13 +1090,10 @@ impl ExtractPolicy for Descriptor<DescriptorPublicKey> {
             build_sat: BuildSatisfaction,
             secp: &SecpCtx,
         ) -> Result<Option<Policy>, Error> {
-            Ok(Policy::make_multisig::<Ctx>(
-                keys.pks.as_ref(),
-                signers,
-                build_sat,
-                keys.k,
-                true,
-                secp,
+            let threshold = Threshold::new(keys.k(), keys.pks().to_vec())
+                .expect("valid threshold and pks collection");
+            Ok(Policy::make_multi::<Ctx, MAX_PUBKEYS_PER_MULTISIG>(
+                &threshold, signers, build_sat, true, secp,
             )?)
         }
 
@@ -1137,7 +1137,7 @@ impl ExtractPolicy for Descriptor<DescriptorPublicKey> {
                 let key_spend_sig =
                     miniscript::Tap::make_signature(tr.internal_key(), signers, build_sat, secp);
 
-                if tr.taptree().is_none() {
+                if tr.tap_tree().is_none() {
                     Ok(Some(key_spend_sig))
                 } else {
                     let mut items = vec![key_spend_sig];
@@ -1184,8 +1184,8 @@ mod test {
         secp: &SecpCtx,
     ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
         let path = bip32::DerivationPath::from_str(path).unwrap();
-        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
-        let tpub = bip32::ExtendedPubKey::from_priv(secp, &tprv);
+        let tprv = bip32::Xpriv::from_str(tprv).unwrap();
+        let tpub = bip32::Xpub::from_priv(secp, &tprv);
         let fingerprint = tprv.fingerprint(secp);
         let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
         let pubkey = (tpub, path).into_descriptor_key().unwrap();

crates/wallet/src/descriptor/template.rs

@@ -36,17 +36,17 @@ pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
 /// ## Example
 ///
 /// ```
-/// use bdk::descriptor::error::Error as DescriptorError;
-/// use bdk::keys::{IntoDescriptorKey, KeyError};
-/// use bdk::miniscript::Legacy;
-/// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
+/// use bdk_wallet::descriptor::error::Error as DescriptorError;
+/// use bdk_wallet::keys::{IntoDescriptorKey, KeyError};
+/// use bdk_wallet::miniscript::Legacy;
+/// use bdk_wallet::template::{DescriptorTemplate, DescriptorTemplateOut};
 /// use bitcoin::Network;
 ///
 /// struct MyP2PKH<K: IntoDescriptorKey<Legacy>>(K);
 ///
 /// impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for MyP2PKH<K> {
 ///     fn build(self, network: Network) -> Result<DescriptorTemplateOut, DescriptorError> {
-///         Ok(bdk::descriptor!(pkh(self.0))?)
+///         Ok(bdk_wallet::descriptor!(pkh(self.0))?)
 ///     }
 /// }
 /// ```
@@ -72,21 +72,28 @@ impl<T: DescriptorTemplate> IntoWalletDescriptor for T {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::P2Pkh;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Pkh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Pkh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::create(P2Pkh(key_external), P2Pkh(key_internal))
+///     .network(Network::Testnet)
+///     .create_wallet_no_persist()?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct P2Pkh<K: IntoDescriptorKey<Legacy>>(pub K);
 
 impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2Pkh<K> {
@@ -100,22 +107,29 @@ impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2Pkh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// use bdk::template::P2Wpkh_P2Sh;
-/// use bdk::wallet::AddressIndex;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Wpkh_P2Sh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Wpkh_P2Sh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::create(P2Wpkh_P2Sh(key_external), P2Wpkh_P2Sh(key_internal))
+///     .network(Network::Testnet)
+///     .create_wallet_no_persist()?;
 ///
 /// assert_eq!(
-///     wallet.get_address(AddressIndex::New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 #[allow(non_camel_case_types)]
+#[derive(Debug, Clone)]
 pub struct P2Wpkh_P2Sh<K: IntoDescriptorKey<Segwitv0>>(pub K);
 
 impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh_P2Sh<K> {
@@ -129,21 +143,28 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh_P2Sh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet};
-/// use bdk::template::P2Wpkh;
-/// use bdk::wallet::AddressIndex::New;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Wpkh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Wpkh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::create(P2Wpkh(key_external), P2Wpkh(key_internal))
+///     .network(Network::Testnet)
+///     .create_wallet_no_persist()?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct P2Wpkh<K: IntoDescriptorKey<Segwitv0>>(pub K);
 
 impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh<K> {
@@ -157,21 +178,28 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::P2TR;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2TR;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2TR(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::create(P2TR(key_external), P2TR(key_internal))
+///     .network(Network::Testnet)
+///     .create_wallet_no_persist()?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "tb1pvjf9t34fznr53u5tqhejz4nr69luzkhlvsdsdfq9pglutrpve2xq7hps46"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct P2TR<K: IntoDescriptorKey<Tap>>(pub K);
 
 impl<K: IntoDescriptorKey<Tap>> DescriptorTemplate for P2TR<K> {
@@ -188,24 +216,22 @@ impl<K: IntoDescriptorKey<Tap>> DescriptorTemplate for P2TR<K> {
 ///
 /// ## Example
 ///
-/// ```
+/// ```rust
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip44;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip44;
 ///
-/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
-///     Bip44(key.clone(), KeychainKind::External),
-///     Some(Bip44(key, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::create(Bip44(key.clone(), KeychainKind::External), Bip44(key, KeychainKind::Internal))
+///     .network(Network::Testnet)
+///     .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "mmogjc7HJEZkrLqyQYqJmxUqFaC7i4uf89");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "pkh([c55b303f/44'/1'/0']tpubDCuorCpzvYS2LCD75BR46KHE8GdDeg1wsAgNZeNr6DaB5gQK1o14uErKwKLuFmeemkQ6N2m3rNgvctdJLyr7nwu2yia7413Hhg8WWE44cgT/0/*)#5wrnv0xt");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "mmogjc7HJEZkrLqyQYqJmxUqFaC7i4uf89");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "pkh([c55b303f/44'/1'/0']tpubDCuorCpzvYS2LCD75BR46KHE8GdDeg1wsAgNZeNr6DaB5gQK1o14uErKwKLuFmeemkQ6N2m3rNgvctdJLyr7nwu2yia7413Hhg8WWE44cgT/0/*)#5wrnv0xt");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
 
 impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
@@ -227,23 +253,24 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip44Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{KeychainKind, Wallet};
+/// use bdk_wallet::template::Bip44Public;
 ///
-/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
 /// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let mut wallet = Wallet::create(
 ///     Bip44Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip44Public(key, fingerprint, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip44Public(key, fingerprint, KeychainKind::Internal),
+///     )
+///     .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "pkh([c55b303f/44'/1'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#cfhumdqz");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "pkh([c55b303f/44'/1'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#cfhumdqz");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
 impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
@@ -265,22 +292,23 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip49;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip49;
 ///
-/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::create(
 ///     Bip49(key.clone(), KeychainKind::External),
-///     Some(Bip49(key, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip49(key, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "2N4zkWAoGdUv4NXhSsU8DvS5MB36T8nKHEB");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDDYr4kdnZgjjShzYNjZUZXUUtpXaofdkMaipyS8ThEh45qFmhT4hKYways7UXmg6V7het1QiFo9kf4kYUXyDvV4rHEyvSpys9pjCB3pukxi/0/*))#s9vxlc8e");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "2N4zkWAoGdUv4NXhSsU8DvS5MB36T8nKHEB");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDDYr4kdnZgjjShzYNjZUZXUUtpXaofdkMaipyS8ThEh45qFmhT4hKYways7UXmg6V7het1QiFo9kf4kYUXyDvV4rHEyvSpys9pjCB3pukxi/0/*))#s9vxlc8e");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
 
 impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
@@ -302,23 +330,24 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip49Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip49Public;
 ///
-/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
 /// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let mut wallet = Wallet::create(
 ///     Bip49Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip49Public(key, fingerprint, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip49Public(key, fingerprint, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#3tka9g0q");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#3tka9g0q");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
 impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
@@ -340,22 +369,23 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip84;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip84;
 ///
-/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::create(
 ///     Bip84(key.clone(), KeychainKind::External),
-///     Some(Bip84(key, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip84(key, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1qhl85z42h7r4su5u37rvvw0gk8j2t3n9y7zsg4n");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "wpkh([c55b303f/84'/1'/0']tpubDDc5mum24DekpNw92t6fHGp8Gr2JjF9J7i4TZBtN6Vp8xpAULG5CFaKsfugWa5imhrQQUZKXe261asP5koDHo5bs3qNTmf3U3o4v9SaB8gg/0/*)#6kfecsmr");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1qhl85z42h7r4su5u37rvvw0gk8j2t3n9y7zsg4n");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "wpkh([c55b303f/84'/1'/0']tpubDDc5mum24DekpNw92t6fHGp8Gr2JjF9J7i4TZBtN6Vp8xpAULG5CFaKsfugWa5imhrQQUZKXe261asP5koDHo5bs3qNTmf3U3o4v9SaB8gg/0/*)#6kfecsmr");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
 
 impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
@@ -377,23 +407,24 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip84Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip84Public;
 ///
-/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
 /// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let mut wallet = Wallet::create(
 ///     Bip84Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip84Public(key, fingerprint, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip84Public(key, fingerprint, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "wpkh([c55b303f/84'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#dhu402yv");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "wpkh([c55b303f/84'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#dhu402yv");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
 impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
@@ -415,22 +446,23 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip86;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip86;
 ///
-/// let key = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::create(
 ///     Bip86(key.clone(), KeychainKind::External),
-///     Some(Bip86(key, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip86(key, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1p5unlj09djx8xsjwe97269kqtxqpwpu2epeskgqjfk4lnf69v4tnqpp35qu");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "tr([c55b303f/86'/1'/0']tpubDCiHofpEs47kx358bPdJmTZHmCDqQ8qw32upCSxHrSEdeeBs2T5Mq6QMB2ukeMqhNBiyhosBvJErteVhfURPGXPv3qLJPw5MVpHUewsbP2m/0/*)#dkgvr5hm");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1p5unlj09djx8xsjwe97269kqtxqpwpu2epeskgqjfk4lnf69v4tnqpp35qu");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "tr([c55b303f/86'/1'/0']tpubDCiHofpEs47kx358bPdJmTZHmCDqQ8qw32upCSxHrSEdeeBs2T5Mq6QMB2ukeMqhNBiyhosBvJErteVhfURPGXPv3qLJPw5MVpHUewsbP2m/0/*)#dkgvr5hm");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip86<K: DerivableKey<Tap>>(pub K, pub KeychainKind);
 
 impl<K: DerivableKey<Tap>> DescriptorTemplate for Bip86<K> {
@@ -452,23 +484,24 @@ impl<K: DerivableKey<Tap>> DescriptorTemplate for Bip86<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip86Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet, KeychainKind};
+/// use bdk_wallet::template::Bip86Public;
 ///
-/// let key = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
 /// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let mut wallet = Wallet::create(
 ///     Bip86Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip86Public(key, fingerprint, KeychainKind::Internal)),
-///     Network::Testnet,
-/// )?;
+///     Bip86Public(key, fingerprint, KeychainKind::Internal),
+/// )
+/// .network(Network::Testnet)
+/// .create_wallet_no_persist()?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1pwjp9f2k5n0xq73ecuu0c5njvgqr3vkh7yaylmpqvsuuaafymh0msvcmh37");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "tr([c55b303f/86'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#2p65srku");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1pwjp9f2k5n0xq73ecuu0c5njvgqr3vkh7yaylmpqvsuuaafymh0msvcmh37");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "tr([c55b303f/86'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#2p65srku");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
+#[derive(Debug, Clone)]
 pub struct Bip86Public<K: DerivableKey<Tap>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
 
 impl<K: DerivableKey<Tap>> DescriptorTemplate for Bip86Public<K> {
@@ -567,8 +600,8 @@ mod test {
     fn test_bip44_template_cointype() {
         use bitcoin::bip32::ChildNumber::{self, Hardened};
 
-        let xprvkey = bitcoin::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
-        assert_eq!(Network::Bitcoin, xprvkey.network);
+        let xprvkey = bitcoin::bip32::Xpriv::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
+        assert!(xprvkey.network.is_mainnet());
         let xdesc = Bip44(xprvkey, KeychainKind::Internal)
             .build(Network::Bitcoin)
             .unwrap();
@@ -581,8 +614,8 @@ mod test {
             assert_matches!(coin_type, Hardened { index: 0 });
         }
 
-        let tprvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
-        assert_eq!(Network::Testnet, tprvkey.network);
+        let tprvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        assert!(!tprvkey.network.is_mainnet());
         let tdesc = Bip44(tprvkey, KeychainKind::Internal)
             .build(Network::Testnet)
             .unwrap();
@@ -740,7 +773,7 @@ mod test {
     // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip44_template() {
-        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip44(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -770,7 +803,7 @@ mod test {
     // BIP44 public `pkh(key/{0,1}/*)`
     #[test]
     fn test_bip44_public_template() {
-        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
         let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip44Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
@@ -801,7 +834,7 @@ mod test {
     // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
     #[test]
     fn test_bip49_template() {
-        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip49(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -831,7 +864,7 @@ mod test {
     // BIP49 public `sh(wpkh(key/{0,1}/*))`
     #[test]
     fn test_bip49_public_template() {
-        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
         let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip49Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
@@ -862,7 +895,7 @@ mod test {
     // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip84_template() {
-        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip84(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -892,7 +925,7 @@ mod test {
     // BIP84 public `wpkh(key/{0,1}/*)`
     #[test]
     fn test_bip84_public_template() {
-        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
         let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip84Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
@@ -924,7 +957,7 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_template() {
-        let prvkey = bitcoin::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
         check(
             Bip86(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -955,7 +988,7 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_public_template() {
-        let pubkey = bitcoin::bip32::ExtendedPubKey::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
         let fingerprint = bitcoin::bip32::Fingerprint::from_str("73c5da0a").unwrap();
         check(
             Bip86Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),

crates/wallet/src/keys/bip39.rs

@@ -57,7 +57,7 @@ pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
 #[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
 impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self[..])?.into())
+        Ok(bip32::Xpriv::new_master(Network::Bitcoin, &self[..])?.into())
     }
 
     fn into_descriptor_key(

crates/wallet/src/keys/mod.rs

@@ -20,6 +20,8 @@ use core::marker::PhantomData;
 use core::ops::Deref;
 use core::str::FromStr;
 
+use rand_core::{CryptoRng, RngCore};
+
 use bitcoin::secp256k1::{self, Secp256k1, Signing};
 
 use bitcoin::bip32;
@@ -97,7 +99,7 @@ impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
         }
     }
 
-    // This method is used internally by `bdk::fragment!` and `bdk::descriptor!`. It has to be
+    // This method is used internally by `bdk_wallet::fragment!` and `bdk_wallet::descriptor!`. It has to be
     // public because it is effectively called by external crates once the macros are expanded,
     // but since it is not meant to be part of the public api we hide it from the docs.
     #[doc(hidden)]
@@ -110,7 +112,7 @@ impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
                 Ok((public, KeyMap::default(), valid_networks))
             }
             DescriptorKey::Secret(secret, valid_networks, _) => {
-                let mut key_map = KeyMap::with_capacity(1);
+                let mut key_map = KeyMap::new();
 
                 let public = secret
                     .to_public(secp)
@@ -206,9 +208,9 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type valid in any context:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
+/// use bdk_wallet::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
 ///
 /// pub struct MyKeyType {
 ///     pubkey: PublicKey,
@@ -224,9 +226,9 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type that is only valid on mainnet:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{
+/// use bdk_wallet::keys::{
 ///     mainnet_network, DescriptorKey, DescriptorPublicKey, IntoDescriptorKey, KeyError,
 ///     ScriptContext, SinglePub, SinglePubKey,
 /// };
@@ -251,9 +253,11 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type that internally encodes in which context it's valid. The context is checked at runtime:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext};
+/// use bdk_wallet::keys::{
+///     DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext,
+/// };
 ///
 /// pub struct MyKeyType {
 ///     is_legacy: bool,
@@ -279,17 +283,17 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// makes the compiler (correctly) fail.
 ///
 /// ```compile_fail
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 /// use core::str::FromStr;
 ///
-/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
+/// use bdk_wallet::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
 ///
 /// pub struct MySegwitOnlyKeyType {
 ///     pubkey: PublicKey,
 /// }
 ///
-/// impl IntoDescriptorKey<bdk::miniscript::Segwitv0> for MySegwitOnlyKeyType {
-///     fn into_descriptor_key(self) -> Result<DescriptorKey<bdk::miniscript::Segwitv0>, KeyError> {
+/// impl IntoDescriptorKey<bdk_wallet::miniscript::Segwitv0> for MySegwitOnlyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<bdk_wallet::miniscript::Segwitv0>, KeyError> {
 ///         self.pubkey.into_descriptor_key()
 ///     }
 /// }
@@ -297,8 +301,8 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// let key = MySegwitOnlyKeyType {
 ///     pubkey: PublicKey::from_str("...")?,
 /// };
-/// let (descriptor, _, _) = bdk::descriptor!(pkh(key))?;
-/// //                                       ^^^^^ changing this to `wpkh` would make it compile
+/// let (descriptor, _, _) = bdk_wallet::descriptor!(pkh(key))?;
+/// //                                               ^^^^^ changing this to `wpkh` would make it compile
 ///
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
@@ -309,15 +313,15 @@ pub trait IntoDescriptorKey<Ctx: ScriptContext>: Sized {
 
 /// Enum for extended keys that can be either `xprv` or `xpub`
 ///
-/// An instance of [`ExtendedKey`] can be constructed from an [`ExtendedPrivKey`](bip32::ExtendedPrivKey)
-/// or an [`ExtendedPubKey`](bip32::ExtendedPubKey) by using the `From` trait.
+/// An instance of [`ExtendedKey`] can be constructed from an [`Xpriv`](bip32::Xpriv)
+/// or an [`Xpub`](bip32::Xpub) by using the `From` trait.
 ///
 /// Defaults to the [`Legacy`](miniscript::Legacy) context.
 pub enum ExtendedKey<Ctx: ScriptContext = miniscript::Legacy> {
     /// A private extended key, aka an `xprv`
-    Private((bip32::ExtendedPrivKey, PhantomData<Ctx>)),
+    Private((bip32::Xpriv, PhantomData<Ctx>)),
     /// A public extended key, aka an `xpub`
-    Public((bip32::ExtendedPubKey, PhantomData<Ctx>)),
+    Public((bip32::Xpub, PhantomData<Ctx>)),
 }
 
 impl<Ctx: ScriptContext> ExtendedKey<Ctx> {
@@ -329,43 +333,43 @@ impl<Ctx: ScriptContext> ExtendedKey<Ctx> {
         }
     }
 
-    /// Transform the [`ExtendedKey`] into an [`ExtendedPrivKey`](bip32::ExtendedPrivKey) for the
+    /// Transform the [`ExtendedKey`] into an [`Xpriv`](bip32::Xpriv) for the
     /// given [`Network`], if the key contains the private data
-    pub fn into_xprv(self, network: Network) -> Option<bip32::ExtendedPrivKey> {
+    pub fn into_xprv(self, network: Network) -> Option<bip32::Xpriv> {
         match self {
             ExtendedKey::Private((mut xprv, _)) => {
-                xprv.network = network;
+                xprv.network = network.into();
                 Some(xprv)
             }
             ExtendedKey::Public(_) => None,
         }
     }
 
-    /// Transform the [`ExtendedKey`] into an [`ExtendedPubKey`](bip32::ExtendedPubKey) for the
+    /// Transform the [`ExtendedKey`] into an [`Xpub`](bip32::Xpub) for the
     /// given [`Network`]
     pub fn into_xpub<C: Signing>(
         self,
         network: bitcoin::Network,
         secp: &Secp256k1<C>,
-    ) -> bip32::ExtendedPubKey {
+    ) -> bip32::Xpub {
         let mut xpub = match self {
-            ExtendedKey::Private((xprv, _)) => bip32::ExtendedPubKey::from_priv(secp, &xprv),
+            ExtendedKey::Private((xprv, _)) => bip32::Xpub::from_priv(secp, &xprv),
             ExtendedKey::Public((xpub, _)) => xpub,
         };
 
-        xpub.network = network;
+        xpub.network = network.into();
         xpub
     }
 }
 
-impl<Ctx: ScriptContext> From<bip32::ExtendedPubKey> for ExtendedKey<Ctx> {
-    fn from(xpub: bip32::ExtendedPubKey) -> Self {
+impl<Ctx: ScriptContext> From<bip32::Xpub> for ExtendedKey<Ctx> {
+    fn from(xpub: bip32::Xpub) -> Self {
         ExtendedKey::Public((xpub, PhantomData))
     }
 }
 
-impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
-    fn from(xprv: bip32::ExtendedPrivKey) -> Self {
+impl<Ctx: ScriptContext> From<bip32::Xpriv> for ExtendedKey<Ctx> {
+    fn from(xprv: bip32::Xpriv) -> Self {
         ExtendedKey::Private((xprv, PhantomData))
     }
 }
@@ -383,13 +387,13 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// ## Examples
 ///
-/// Key types that can be directly converted into an [`ExtendedPrivKey`] or
-/// an [`ExtendedPubKey`] can implement only the required `into_extended_key()` method.
+/// Key types that can be directly converted into an [`Xpriv`] or
+/// an [`Xpub`] can implement only the required `into_extended_key()` method.
 ///
 /// ```
-/// use bdk::bitcoin;
-/// use bdk::bitcoin::bip32;
-/// use bdk::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
+/// use bdk_wallet::bitcoin;
+/// use bdk_wallet::bitcoin::bip32;
+/// use bdk_wallet::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
 ///
 /// struct MyCustomKeyType {
 ///     key_data: bitcoin::PrivateKey,
@@ -399,8 +403,8 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
 ///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-///         let xprv = bip32::ExtendedPrivKey {
-///             network: self.network,
+///         let xprv = bip32::Xpriv {
+///             network: self.network.into(),
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
@@ -415,12 +419,12 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// Types that don't internally encode the [`Network`] in which they are valid need some extra
 /// steps to override the set of valid networks, otherwise only the network specified in the
-/// [`ExtendedPrivKey`] or [`ExtendedPubKey`] will be considered valid.
+/// [`Xpriv`] or [`Xpub`] will be considered valid.
 ///
 /// ```
-/// use bdk::bitcoin;
-/// use bdk::bitcoin::bip32;
-/// use bdk::keys::{
+/// use bdk_wallet::bitcoin;
+/// use bdk_wallet::bitcoin::bip32;
+/// use bdk_wallet::keys::{
 ///     any_network, DerivableKey, DescriptorKey, ExtendedKey, KeyError, ScriptContext,
 /// };
 ///
@@ -431,8 +435,8 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
 ///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-///         let xprv = bip32::ExtendedPrivKey {
-///             network: bitcoin::Network::Bitcoin, // pick an arbitrary network here
+///         let xprv = bip32::Xpriv {
+///             network: bitcoin::Network::Bitcoin.into(), // pick an arbitrary network here
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
@@ -459,8 +463,8 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 /// ```
 ///
 /// [`DerivationPath`]: (bip32::DerivationPath)
-/// [`ExtendedPrivKey`]: (bip32::ExtendedPrivKey)
-/// [`ExtendedPubKey`]: (bip32::ExtendedPubKey)
+/// [`Xpriv`]: (bip32::Xpriv)
+/// [`Xpub`]: (bip32::Xpub)
 pub trait DerivableKey<Ctx: ScriptContext = miniscript::Legacy>: Sized {
     /// Consume `self` and turn it into an [`ExtendedKey`]
     #[cfg_attr(
@@ -469,9 +473,9 @@ pub trait DerivableKey<Ctx: ScriptContext = miniscript::Legacy>: Sized {
 This can be used to get direct access to `xprv`s and `xpub`s for types that implement this trait,
 like [`Mnemonic`](bip39::Mnemonic) when the `keys-bip39` feature is enabled.
 ```rust
-use bdk::bitcoin::Network;
-use bdk::keys::{DerivableKey, ExtendedKey};
-use bdk::keys::bip39::{Mnemonic, Language};
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::keys::{DerivableKey, ExtendedKey};
+use bdk_wallet::keys::bip39::{Mnemonic, Language};
 
 # fn main() -> Result<(), Box<dyn std::error::Error>> {
 let xkey: ExtendedKey =
@@ -520,13 +524,13 @@ impl<Ctx: ScriptContext> DerivableKey<Ctx> for ExtendedKey<Ctx> {
     }
 }
 
-impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPubKey {
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::Xpub {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
         Ok(self.into())
     }
 }
 
-impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPrivKey {
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::Xpriv {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
         Ok(self.into())
     }
@@ -629,12 +633,23 @@ pub trait GeneratableKey<Ctx: ScriptContext>: Sized {
         entropy: Self::Entropy,
     ) -> Result<GeneratedKey<Self, Ctx>, Self::Error>;
 
-    /// Generate a key given the options with a random entropy
+    /// Generate a key given the options with random entropy.
+    ///
+    /// Uses the thread-local random number generator.
+    #[cfg(feature = "std")]
     fn generate(options: Self::Options) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
-        use rand::{thread_rng, Rng};
+        Self::generate_with_aux_rand(options, &mut bitcoin::key::rand::thread_rng())
+    }
 
+    /// Generate a key given the options with random entropy.
+    ///
+    /// Uses a provided random number generator (rng).
+    fn generate_with_aux_rand(
+        options: Self::Options,
+        rng: &mut (impl CryptoRng + RngCore),
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
         let mut entropy = Self::Entropy::default();
-        thread_rng().fill(entropy.as_mut());
+        rng.fill_bytes(entropy.as_mut());
         Self::generate_with_entropy(options, entropy)
     }
 }
@@ -655,8 +670,20 @@ where
     }
 
     /// Generate a key with the default options and a random entropy
+    ///
+    /// Uses the thread-local random number generator.
+    #[cfg(feature = "std")]
     fn generate_default() -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
-        Self::generate(Default::default())
+        Self::generate_with_aux_rand(Default::default(), &mut bitcoin::key::rand::thread_rng())
+    }
+
+    /// Generate a key with the default options and a random entropy
+    ///
+    /// Uses a provided random number generator (rng).
+    fn generate_default_with_aux_rand(
+        rng: &mut (impl CryptoRng + RngCore),
+    ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
+        Self::generate_with_aux_rand(Default::default(), rng)
     }
 }
 
@@ -670,7 +697,7 @@ where
 {
 }
 
-impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::Xpriv {
     type Entropy = [u8; 32];
 
     type Options = ();
@@ -681,7 +708,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
         entropy: Self::Entropy,
     ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
         // pick a arbitrary network here, but say that we support all of them
-        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, entropy.as_ref())?;
+        let xprv = bip32::Xpriv::new_master(Network::Bitcoin, entropy.as_ref())?;
         Ok(GeneratedKey::new(xprv, any_network()))
     }
 }
@@ -715,7 +742,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for PrivateKey {
         let inner = secp256k1::SecretKey::from_slice(&entropy)?;
         let private_key = PrivateKey {
             compressed: options.compressed,
-            network: Network::Bitcoin,
+            network: Network::Bitcoin.into(),
             inner,
         };
 
@@ -764,7 +791,7 @@ fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((pks, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `pk_k()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `pk_k()` fragments
 #[doc(hidden)]
 pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     descriptor_key: Pk,
@@ -778,7 +805,7 @@ pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `pk_h()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `pk_h()` fragments
 #[doc(hidden)]
 pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     descriptor_key: Pk,
@@ -792,7 +819,7 @@ pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `multi()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `multi()` fragments
 #[doc(hidden)]
 pub fn make_multi<
     Pk: IntoDescriptorKey<Ctx>,
@@ -812,7 +839,7 @@ pub fn make_multi<
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::descriptor!` to build `sortedmulti()` fragments
+// Used internally by `bdk_wallet::descriptor!` to build `sortedmulti()` fragments
 #[doc(hidden)]
 pub fn make_sortedmulti<Pk, Ctx, F>(
     thresh: usize,
@@ -834,7 +861,7 @@ where
     Ok((descriptor, key_map, valid_networks))
 }
 
-/// The "identity" conversion is used internally by some `bdk::fragment`s
+/// The "identity" conversion is used internally by some `bdk_wallet::fragment`s
 impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorKey<Ctx> {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         Ok(self)
@@ -845,9 +872,7 @@ impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorPublicKey {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         let networks = match self {
             DescriptorPublicKey::Single(_) => any_network(),
-            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. })
-                if xkey.network == Network::Bitcoin =>
-            {
+            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. }) if xkey.network.is_mainnet() => {
                 mainnet_network()
             }
             _ => test_networks(),
@@ -880,12 +905,8 @@ impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for XOnlyPublicKey {
 impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorSecretKey {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         let networks = match &self {
-            DescriptorSecretKey::Single(sk) if sk.key.network == Network::Bitcoin => {
-                mainnet_network()
-            }
-            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. })
-                if xkey.network == Network::Bitcoin =>
-            {
+            DescriptorSecretKey::Single(sk) if sk.key.network.is_mainnet() => mainnet_network(),
+            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. }) if xkey.network.is_mainnet() => {
                 mainnet_network()
             }
             _ => test_networks(),
@@ -914,7 +935,7 @@ impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for PrivateKey {
 }
 
 /// Errors thrown while working with [`keys`](crate::keys)
-#[derive(Debug)]
+#[derive(Debug, PartialEq)]
 pub enum KeyError {
     /// The key cannot exist in the given script context
     InvalidScriptContext,
@@ -971,7 +992,7 @@ pub mod test {
     #[test]
     fn test_keys_generate_xprv() {
         let generated_xprv: GeneratedKey<_, miniscript::Segwitv0> =
-            bip32::ExtendedPrivKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+            bip32::Xpriv::generate_with_entropy_default(TEST_ENTROPY).unwrap();
 
         assert_eq!(generated_xprv.valid_networks, any_network());
         assert_eq!(generated_xprv.to_string(), "xprv9s21ZrQH143K4Xr1cJyqTvuL2FWR8eicgY9boWqMBv8MDVUZ65AXHnzBrK1nyomu6wdcabRgmGTaAKawvhAno1V5FowGpTLVx3jxzE5uk3Q");
@@ -1001,6 +1022,6 @@ pub mod test {
         .unwrap();
         let xprv = xkey.into_xprv(Network::Testnet).unwrap();
 
-        assert_eq!(xprv.network, Network::Testnet);
+        assert_eq!(xprv.network, Network::Testnet.into());
     }
 }

crates/wallet/src/lib.rs

@@ -15,33 +15,36 @@ extern crate std;
 #[doc(hidden)]
 #[macro_use]
 pub extern crate alloc;
-
+pub extern crate bdk_chain as chain;
+#[cfg(feature = "file_store")]
+pub extern crate bdk_file_store as file_store;
+#[cfg(feature = "keys-bip39")]
+pub extern crate bip39;
 pub extern crate bitcoin;
 pub extern crate miniscript;
-extern crate serde;
-extern crate serde_json;
-
-#[cfg(feature = "keys-bip39")]
-extern crate bip39;
+pub extern crate serde;
+pub extern crate serde_json;
 
 pub mod descriptor;
 pub mod keys;
 pub mod psbt;
-pub(crate) mod types;
-pub mod wallet;
+mod types;
+mod wallet;
 
+pub(crate) use bdk_chain::collections;
+#[cfg(feature = "rusqlite")]
+pub use bdk_chain::rusqlite;
+#[cfg(feature = "rusqlite")]
+pub use bdk_chain::rusqlite_impl;
 pub use descriptor::template;
 pub use descriptor::HdKeyPaths;
+pub use signer;
+pub use signer::SignOptions;
+pub use tx_builder::*;
 pub use types::*;
-pub use wallet::signer;
-pub use wallet::signer::SignOptions;
-pub use wallet::tx_builder::TxBuilder;
-pub use wallet::Wallet;
+pub use wallet::*;
 
-/// Get the version of BDK at runtime
+/// Get the version of [`bdk_wallet`](crate) at runtime.
 pub fn version() -> &'static str {
     env!("CARGO_PKG_VERSION", "unknown")
 }
-
-pub use bdk_chain as chain;
-pub(crate) use bdk_chain::collections;

crates/wallet/src/psbt/mod.rs

@@ -9,11 +9,12 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-//! Additional functions on the `rust-bitcoin` `PartiallySignedTransaction` structure.
+//! Additional functions on the `rust-bitcoin` `Psbt` structure.
 
-use crate::FeeRate;
 use alloc::vec::Vec;
-use bitcoin::psbt::PartiallySignedTransaction as Psbt;
+use bitcoin::Amount;
+use bitcoin::FeeRate;
+use bitcoin::Psbt;
 use bitcoin::TxOut;
 
 // TODO upstream the functions here to `rust-bitcoin`?
@@ -25,10 +26,10 @@ pub trait PsbtUtils {
 
     /// The total transaction fee amount, sum of input amounts minus sum of output amounts, in sats.
     /// If the PSBT is missing a TxOut for an input returns None.
-    fn fee_amount(&self) -> Option<u64>;
+    fn fee_amount(&self) -> Option<Amount>;
 
     /// The transaction's fee rate. This value will only be accurate if calculated AFTER the
-    /// `PartiallySignedTransaction` is finalized and all witness/signature data is added to the
+    /// `Psbt` is finalized and all witness/signature data is added to the
     /// transaction.
     /// If the PSBT is missing a TxOut for an input returns None.
     fn fee_rate(&self) -> Option<FeeRate>;
@@ -48,13 +49,13 @@ impl PsbtUtils for Psbt {
         }
     }
 
-    fn fee_amount(&self) -> Option<u64> {
+    fn fee_amount(&self) -> Option<Amount> {
         let tx = &self.unsigned_tx;
         let utxos: Option<Vec<TxOut>> = (0..tx.input.len()).map(|i| self.get_utxo_for(i)).collect();
 
         utxos.map(|inputs| {
-            let input_amount: u64 = inputs.iter().map(|i| i.value).sum();
-            let output_amount: u64 = self.unsigned_tx.output.iter().map(|o| o.value).sum();
+            let input_amount: Amount = inputs.iter().map(|i| i.value).sum();
+            let output_amount: Amount = self.unsigned_tx.output.iter().map(|o| o.value).sum();
             input_amount
                 .checked_sub(output_amount)
                 .expect("input amount must be greater than output amount")
@@ -63,9 +64,7 @@ impl PsbtUtils for Psbt {
 
     fn fee_rate(&self) -> Option<FeeRate> {
         let fee_amount = self.fee_amount();
-        fee_amount.map(|fee| {
-            let weight = self.clone().extract_tx().weight();
-            FeeRate::from_wu(fee, weight)
-        })
+        let weight = self.clone().extract_tx().ok()?.weight();
+        fee_amount.map(|fee| fee / weight)
     }
 }

crates/wallet/src/types.rs

@@ -0,0 +1,135 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use alloc::boxed::Box;
+use core::convert::AsRef;
+
+use bdk_chain::ConfirmationTime;
+use bitcoin::transaction::{OutPoint, Sequence, TxOut};
+use bitcoin::{psbt, Weight};
+
+use serde::{Deserialize, Serialize};
+
+/// Types of keychains
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+pub enum KeychainKind {
+    /// External keychain, used for deriving recipient addresses.
+    External = 0,
+    /// Internal keychain, used for deriving change addresses.
+    Internal = 1,
+}
+
+impl KeychainKind {
+    /// Return [`KeychainKind`] as a byte
+    pub fn as_byte(&self) -> u8 {
+        match self {
+            KeychainKind::External => b'e',
+            KeychainKind::Internal => b'i',
+        }
+    }
+}
+
+impl AsRef<[u8]> for KeychainKind {
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            KeychainKind::External => b"e",
+            KeychainKind::Internal => b"i",
+        }
+    }
+}
+
+/// An unspent output owned by a [`Wallet`].
+///
+/// [`Wallet`]: crate::Wallet
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LocalOutput {
+    /// Reference to a transaction output
+    pub outpoint: OutPoint,
+    /// Transaction output
+    pub txout: TxOut,
+    /// Type of keychain
+    pub keychain: KeychainKind,
+    /// Whether this UTXO is spent or not
+    pub is_spent: bool,
+    /// The derivation index for the script pubkey in the wallet
+    pub derivation_index: u32,
+    /// The confirmation time for transaction containing this utxo
+    pub confirmation_time: ConfirmationTime,
+}
+
+/// A [`Utxo`] with its `satisfaction_weight`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WeightedUtxo {
+    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
+    /// properly maintain the feerate when adding this input to a transaction during coin selection.
+    ///
+    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
+    pub satisfaction_weight: Weight,
+    /// The UTXO
+    pub utxo: Utxo,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+/// An unspent transaction output (UTXO).
+pub enum Utxo {
+    /// A UTXO owned by the local wallet.
+    Local(LocalOutput),
+    /// A UTXO owned by another wallet.
+    Foreign {
+        /// The location of the output.
+        outpoint: OutPoint,
+        /// The nSequence value to set for this input.
+        sequence: Option<Sequence>,
+        /// The information about the input we require to add it to a PSBT.
+        // Box it to stop the type being too big.
+        psbt_input: Box<psbt::Input>,
+    },
+}
+
+impl Utxo {
+    /// Get the location of the UTXO
+    pub fn outpoint(&self) -> OutPoint {
+        match &self {
+            Utxo::Local(local) => local.outpoint,
+            Utxo::Foreign { outpoint, .. } => *outpoint,
+        }
+    }
+
+    /// Get the `TxOut` of the UTXO
+    pub fn txout(&self) -> &TxOut {
+        match &self {
+            Utxo::Local(local) => &local.txout,
+            Utxo::Foreign {
+                outpoint,
+                psbt_input,
+                ..
+            } => {
+                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+                    return &prev_tx.output[outpoint.vout as usize];
+                }
+
+                if let Some(txout) = &psbt_input.witness_utxo {
+                    return txout;
+                }
+
+                unreachable!("Foreign UTXOs will always have one of these set")
+            }
+        }
+    }
+
+    /// Get the sequence number if an explicit sequence number has to be set for this input.
+    pub fn sequence(&self) -> Option<Sequence> {
+        match self {
+            Utxo::Local(_) => None,
+            Utxo::Foreign { sequence, .. } => *sequence,
+        }
+    }
+}

crates/wallet/src/wallet/changeset.rs

@@ -0,0 +1,210 @@
+use bdk_chain::{
+    indexed_tx_graph, keychain_txout, local_chain, tx_graph, ConfirmationBlockTime, Merge,
+};
+use miniscript::{Descriptor, DescriptorPublicKey};
+
+type IndexedTxGraphChangeSet =
+    indexed_tx_graph::ChangeSet<ConfirmationBlockTime, keychain_txout::ChangeSet>;
+
+/// A changeset for [`Wallet`](crate::Wallet).
+#[derive(Default, Debug, Clone, PartialEq, serde::Deserialize, serde::Serialize)]
+#[non_exhaustive]
+pub struct ChangeSet {
+    /// Descriptor for recipient addresses.
+    pub descriptor: Option<Descriptor<DescriptorPublicKey>>,
+    /// Descriptor for change addresses.
+    pub change_descriptor: Option<Descriptor<DescriptorPublicKey>>,
+    /// Stores the network type of the transaction data.
+    pub network: Option<bitcoin::Network>,
+    /// Changes to the [`LocalChain`](local_chain::LocalChain).
+    pub local_chain: local_chain::ChangeSet,
+    /// Changes to [`TxGraph`](tx_graph::TxGraph).
+    pub tx_graph: tx_graph::ChangeSet<ConfirmationBlockTime>,
+    /// Changes to [`KeychainTxOutIndex`](keychain_txout::KeychainTxOutIndex).
+    pub indexer: keychain_txout::ChangeSet,
+}
+
+impl Merge for ChangeSet {
+    /// Merge another [`ChangeSet`] into itself.
+    fn merge(&mut self, other: Self) {
+        if other.descriptor.is_some() {
+            debug_assert!(
+                self.descriptor.is_none() || self.descriptor == other.descriptor,
+                "descriptor must never change"
+            );
+            self.descriptor = other.descriptor;
+        }
+        if other.change_descriptor.is_some() {
+            debug_assert!(
+                self.change_descriptor.is_none()
+                    || self.change_descriptor == other.change_descriptor,
+                "change descriptor must never change"
+            );
+            self.change_descriptor = other.change_descriptor;
+        }
+        if other.network.is_some() {
+            debug_assert!(
+                self.network.is_none() || self.network == other.network,
+                "network must never change"
+            );
+            self.network = other.network;
+        }
+
+        Merge::merge(&mut self.local_chain, other.local_chain);
+        Merge::merge(&mut self.tx_graph, other.tx_graph);
+        Merge::merge(&mut self.indexer, other.indexer);
+    }
+
+    fn is_empty(&self) -> bool {
+        self.descriptor.is_none()
+            && self.change_descriptor.is_none()
+            && self.network.is_none()
+            && self.local_chain.is_empty()
+            && self.tx_graph.is_empty()
+            && self.indexer.is_empty()
+    }
+}
+
+#[cfg(feature = "rusqlite")]
+impl ChangeSet {
+    /// Schema name for wallet.
+    pub const WALLET_SCHEMA_NAME: &'static str = "bdk_wallet";
+    /// Name of table to store wallet descriptors and network.
+    pub const WALLET_TABLE_NAME: &'static str = "bdk_wallet";
+
+    /// Initialize sqlite tables for wallet schema & table.
+    fn init_wallet_sqlite_tables(
+        db_tx: &chain::rusqlite::Transaction,
+    ) -> chain::rusqlite::Result<()> {
+        let schema_v0: &[&str] = &[&format!(
+            "CREATE TABLE {} ( \
+                id INTEGER PRIMARY KEY NOT NULL CHECK (id = 0), \
+                descriptor TEXT, \
+                change_descriptor TEXT, \
+                network TEXT \
+                ) STRICT;",
+            Self::WALLET_TABLE_NAME,
+        )];
+        crate::rusqlite_impl::migrate_schema(db_tx, Self::WALLET_SCHEMA_NAME, &[schema_v0])
+    }
+
+    /// Recover a [`ChangeSet`] from sqlite database.
+    pub fn from_sqlite(db_tx: &chain::rusqlite::Transaction) -> chain::rusqlite::Result<Self> {
+        Self::init_wallet_sqlite_tables(db_tx)?;
+        use chain::rusqlite::OptionalExtension;
+        use chain::Impl;
+        use miniscript::{Descriptor, DescriptorPublicKey};
+
+        let mut changeset = Self::default();
+
+        let mut wallet_statement = db_tx.prepare(&format!(
+            "SELECT descriptor, change_descriptor, network FROM {}",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        let row = wallet_statement
+            .query_row([], |row| {
+                Ok((
+                    row.get::<_, Impl<Descriptor<DescriptorPublicKey>>>("descriptor")?,
+                    row.get::<_, Impl<Descriptor<DescriptorPublicKey>>>("change_descriptor")?,
+                    row.get::<_, Impl<bitcoin::Network>>("network")?,
+                ))
+            })
+            .optional()?;
+        if let Some((Impl(desc), Impl(change_desc), Impl(network))) = row {
+            changeset.descriptor = Some(desc);
+            changeset.change_descriptor = Some(change_desc);
+            changeset.network = Some(network);
+        }
+
+        changeset.local_chain = local_chain::ChangeSet::from_sqlite(db_tx)?;
+        changeset.tx_graph = tx_graph::ChangeSet::<_>::from_sqlite(db_tx)?;
+        changeset.indexer = keychain_txout::ChangeSet::from_sqlite(db_tx)?;
+
+        Ok(changeset)
+    }
+
+    /// Persist [`ChangeSet`] to sqlite database.
+    pub fn persist_to_sqlite(
+        &self,
+        db_tx: &chain::rusqlite::Transaction,
+    ) -> chain::rusqlite::Result<()> {
+        Self::init_wallet_sqlite_tables(db_tx)?;
+        use chain::rusqlite::named_params;
+        use chain::Impl;
+
+        let mut descriptor_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, descriptor) VALUES(:id, :descriptor) ON CONFLICT(id) DO UPDATE SET descriptor=:descriptor",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(descriptor) = &self.descriptor {
+            descriptor_statement.execute(named_params! {
+                ":id": 0,
+                ":descriptor": Impl(descriptor.clone()),
+            })?;
+        }
+
+        let mut change_descriptor_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, change_descriptor) VALUES(:id, :change_descriptor) ON CONFLICT(id) DO UPDATE SET change_descriptor=:change_descriptor",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(change_descriptor) = &self.change_descriptor {
+            change_descriptor_statement.execute(named_params! {
+                ":id": 0,
+                ":change_descriptor": Impl(change_descriptor.clone()),
+            })?;
+        }
+
+        let mut network_statement = db_tx.prepare_cached(&format!(
+            "INSERT INTO {}(id, network) VALUES(:id, :network) ON CONFLICT(id) DO UPDATE SET network=:network",
+            Self::WALLET_TABLE_NAME,
+        ))?;
+        if let Some(network) = self.network {
+            network_statement.execute(named_params! {
+                ":id": 0,
+                ":network": Impl(network),
+            })?;
+        }
+
+        self.local_chain.persist_to_sqlite(db_tx)?;
+        self.tx_graph.persist_to_sqlite(db_tx)?;
+        self.indexer.persist_to_sqlite(db_tx)?;
+        Ok(())
+    }
+}
+
+impl From<local_chain::ChangeSet> for ChangeSet {
+    fn from(chain: local_chain::ChangeSet) -> Self {
+        Self {
+            local_chain: chain,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<IndexedTxGraphChangeSet> for ChangeSet {
+    fn from(indexed_tx_graph: IndexedTxGraphChangeSet) -> Self {
+        Self {
+            tx_graph: indexed_tx_graph.tx_graph,
+            indexer: indexed_tx_graph.indexer,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<tx_graph::ChangeSet<ConfirmationBlockTime>> for ChangeSet {
+    fn from(tx_graph: tx_graph::ChangeSet<ConfirmationBlockTime>) -> Self {
+        Self {
+            tx_graph,
+            ..Default::default()
+        }
+    }
+}
+
+impl From<keychain_txout::ChangeSet> for ChangeSet {
+    fn from(indexer: keychain_txout::ChangeSet) -> Self {
+        Self {
+            indexer,
+            ..Default::default()
+        }
+    }
+}

crates/wallet/src/wallet/coin_selection.rs

@@ -26,13 +26,11 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::{self, ChangeSet, coin_selection::*, coin_selection};
-//! # use bdk::wallet::error::CreateTxError;
-//! # use bdk_chain::PersistBackend;
-//! # use bdk::*;
-//! # use bdk::wallet::coin_selection::decide_change;
+//! # use bdk_wallet::{self, ChangeSet, coin_selection::*, coin_selection};
+//! # use bdk_wallet::error::CreateTxError;
+//! # use bdk_wallet::*;
+//! # use bdk_wallet::coin_selection::decide_change;
 //! # use anyhow::Error;
-//! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
 //! #[derive(Debug)]
 //! struct AlwaysSpendEverything;
 //!
@@ -41,7 +39,7 @@
 //!         &self,
 //!         required_utxos: Vec<WeightedUtxo>,
 //!         optional_utxos: Vec<WeightedUtxo>,
-//!         fee_rate: bdk::FeeRate,
+//!         fee_rate: FeeRate,
 //!         target_amount: u64,
 //!         drain_script: &Script,
 //!     ) -> Result<CoinSelectionResult, coin_selection::Error> {
@@ -53,15 +51,16 @@
 //!             .scan(
 //!                 (&mut selected_amount, &mut additional_weight),
 //!                 |(selected_amount, additional_weight), weighted_utxo| {
-//!                     **selected_amount += weighted_utxo.utxo.txout().value;
-//!                     **additional_weight += Weight::from_wu(
-//!                         (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
-//!                     );
+//!                     **selected_amount += weighted_utxo.utxo.txout().value.to_sat();
+//!                     **additional_weight += TxIn::default()
+//!                         .segwit_weight()
+//!                         .checked_add(weighted_utxo.satisfaction_weight)
+//!                         .expect("`Weight` addition should not cause an integer overflow");
 //!                     Some(weighted_utxo.utxo)
 //!                 },
 //!             )
 //!             .collect::<Vec<_>>();
-//!         let additional_fees = fee_rate.fee_wu(additional_weight);
+//!         let additional_fees = (fee_rate * additional_weight).to_sat();
 //!         let amount_needed_with_fees = additional_fees + target_amount;
 //!         if selected_amount < amount_needed_with_fees {
 //!             return Err(coin_selection::Error::InsufficientFunds {
@@ -91,7 +90,7 @@
 //!     .unwrap();
 //! let psbt = {
 //!     let mut builder = wallet.build_tx().coin_selection(AlwaysSpendEverything);
-//!     builder.add_recipient(to_address.script_pubkey(), 50_000);
+//!     builder.add_recipient(to_address.script_pubkey(), Amount::from_sat(50_000));
 //!     builder.finish()?
 //! };
 //!
@@ -101,28 +100,26 @@
 //! ```
 
 use crate::chain::collections::HashSet;
-use crate::types::FeeRate;
 use crate::wallet::utils::IsDust;
 use crate::Utxo;
 use crate::WeightedUtxo;
+use bitcoin::FeeRate;
 
 use alloc::vec::Vec;
 use bitcoin::consensus::encode::serialize;
 use bitcoin::OutPoint;
+use bitcoin::TxIn;
 use bitcoin::{Script, Weight};
 
 use core::convert::TryInto;
 use core::fmt::{self, Formatter};
-use rand::seq::SliceRandom;
+use rand_core::RngCore;
 
+use super::utils::shuffle_slice;
 /// Default coin selection algorithm used by [`TxBuilder`](super::tx_builder::TxBuilder) if not
 /// overridden
 pub type DefaultCoinSelectionAlgorithm = BranchAndBoundCoinSelection;
 
-// Base weight of a Txin, not counting the weight needed for satisfying it.
-// prev_txid (32 bytes) + prev_vout (4 bytes) + sequence (4 bytes)
-pub(crate) const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
-
 /// Errors that can be thrown by the [`coin_selection`](crate::wallet::coin_selection) module
 #[derive(Debug)]
 pub enum Error {
@@ -195,7 +192,7 @@ pub struct CoinSelectionResult {
 impl CoinSelectionResult {
     /// The total value of the inputs selected.
     pub fn selected_amount(&self) -> u64 {
-        self.selected.iter().map(|u| u.txout().value).sum()
+        self.selected.iter().map(|u| u.txout().value.to_sat()).sum()
     }
 
     /// The total value of the inputs selected from the local wallet.
@@ -203,7 +200,7 @@ impl CoinSelectionResult {
         self.selected
             .iter()
             .filter_map(|u| match u {
-                Utxo::Local(_) => Some(u.txout().value),
+                Utxo::Local(_) => Some(u.txout().value.to_sat()),
                 _ => None,
             })
             .sum()
@@ -313,11 +310,12 @@ impl CoinSelectionAlgorithm for OldestFirstCoinSelection {
 pub fn decide_change(remaining_amount: u64, fee_rate: FeeRate, drain_script: &Script) -> Excess {
     // drain_output_len = size(len(script_pubkey)) + len(script_pubkey) + size(output_value)
     let drain_output_len = serialize(drain_script).len() + 8usize;
-    let change_fee = fee_rate.fee_vb(drain_output_len);
+    let change_fee =
+        (fee_rate * Weight::from_vb(drain_output_len as u64).expect("overflow occurred")).to_sat();
     let drain_val = remaining_amount.saturating_sub(change_fee);
 
     if drain_val.is_dust(drain_script) {
-        let dust_threshold = drain_script.dust_value().to_sat();
+        let dust_threshold = drain_script.minimal_non_dust().to_sat();
         Excess::NoChange {
             dust_threshold,
             change_fee,
@@ -344,10 +342,13 @@ fn select_sorted_utxos(
             (&mut selected_amount, &mut fee_amount),
             |(selected_amount, fee_amount), (must_use, weighted_utxo)| {
                 if must_use || **selected_amount < target_amount + **fee_amount {
-                    **fee_amount += fee_rate.fee_wu(Weight::from_wu(
-                        (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
-                    ));
-                    **selected_amount += weighted_utxo.utxo.txout().value;
+                    **fee_amount += (fee_rate
+                        * (TxIn::default()
+                            .segwit_weight()
+                            .checked_add(weighted_utxo.satisfaction_weight)
+                            .expect("`Weight` addition should not cause an integer overflow")))
+                    .to_sat();
+                    **selected_amount += weighted_utxo.utxo.txout().value.to_sat();
                     Some(weighted_utxo.utxo)
                 } else {
                     None
@@ -387,10 +388,13 @@ struct OutputGroup {
 
 impl OutputGroup {
     fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
-        let fee = fee_rate.fee_wu(Weight::from_wu(
-            (TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight) as u64,
-        ));
-        let effective_value = weighted_utxo.utxo.txout().value as i64 - fee as i64;
+        let fee = (fee_rate
+            * (TxIn::default()
+                .segwit_weight()
+                .checked_add(weighted_utxo.satisfaction_weight)
+                .expect("`Weight` addition should not cause an integer overflow")))
+        .to_sat();
+        let effective_value = weighted_utxo.utxo.txout().value.to_sat() as i64 - fee as i64;
         OutputGroup {
             weighted_utxo,
             fee,
@@ -456,7 +460,8 @@ impl CoinSelectionAlgorithm for BranchAndBoundCoinSelection {
             .iter()
             .fold(0, |acc, x| acc + x.effective_value);
 
-        let cost_of_change = self.size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change =
+            (Weight::from_vb(self.size_of_change).expect("overflow occurred") * fee_rate).to_sat();
 
         // `curr_value` and `curr_available_value` are both the sum of *effective_values* of
         // the UTXOs. For the optional UTXOs (curr_available_value) we filter out UTXOs with
@@ -480,7 +485,7 @@ impl CoinSelectionAlgorithm for BranchAndBoundCoinSelection {
                     .chain(optional_utxos.iter())
                     .fold((0, 0), |(mut fees, mut value), utxo| {
                         fees += utxo.fee;
-                        value += utxo.weighted_utxo.utxo.txout().value;
+                        value += utxo.weighted_utxo.utxo.txout().value.to_sat();
 
                         (fees, value)
                     });
@@ -512,27 +517,16 @@ impl CoinSelectionAlgorithm for BranchAndBoundCoinSelection {
             ));
         }
 
-        Ok(self
-            .bnb(
-                required_utxos.clone(),
-                optional_utxos.clone(),
-                curr_value,
-                curr_available_value,
-                target_amount,
-                cost_of_change,
-                drain_script,
-                fee_rate,
-            )
-            .unwrap_or_else(|_| {
-                self.single_random_draw(
-                    required_utxos,
-                    optional_utxos,
-                    curr_value,
-                    target_amount,
-                    drain_script,
-                    fee_rate,
-                )
-            }))
+        self.bnb(
+            required_utxos.clone(),
+            optional_utxos.clone(),
+            curr_value,
+            curr_available_value,
+            target_amount,
+            cost_of_change,
+            drain_script,
+            fee_rate,
+        )
     }
 }
 
@@ -547,7 +541,7 @@ impl BranchAndBoundCoinSelection {
         mut curr_value: i64,
         mut curr_available_value: i64,
         target_amount: i64,
-        cost_of_change: f32,
+        cost_of_change: u64,
         drain_script: &Script,
         fee_rate: FeeRate,
     ) -> Result<CoinSelectionResult, Error> {
@@ -584,7 +578,7 @@ impl BranchAndBoundCoinSelection {
                 // If we found a solution better than the previous one, or if there wasn't previous
                 // solution, update the best solution
                 if best_selection_value.is_none() || curr_value < best_selection_value.unwrap() {
-                    best_selection = current_selection.clone();
+                    best_selection.clone_from(&current_selection);
                     best_selection_value = Some(curr_value);
                 }
 
@@ -659,40 +653,6 @@ impl BranchAndBoundCoinSelection {
         ))
     }
 
-    #[allow(clippy::too_many_arguments)]
-    fn single_random_draw(
-        &self,
-        required_utxos: Vec<OutputGroup>,
-        mut optional_utxos: Vec<OutputGroup>,
-        curr_value: i64,
-        target_amount: i64,
-        drain_script: &Script,
-        fee_rate: FeeRate,
-    ) -> CoinSelectionResult {
-        optional_utxos.shuffle(&mut rand::thread_rng());
-        let selected_utxos = optional_utxos.into_iter().fold(
-            (curr_value, vec![]),
-            |(mut amount, mut utxos), utxo| {
-                if amount >= target_amount {
-                    (amount, utxos)
-                } else {
-                    amount += utxo.effective_value;
-                    utxos.push(utxo);
-                    (amount, utxos)
-                }
-            },
-        );
-
-        // remaining_amount can't be negative as that would mean the
-        // selection wasn't successful
-        // target_amount = amount_needed + (fee_amount - vin_fees)
-        let remaining_amount = (selected_utxos.0 - target_amount) as u64;
-
-        let excess = decide_change(remaining_amount, fee_rate, drain_script);
-
-        BranchAndBoundCoinSelection::calculate_cs_result(selected_utxos.1, required_utxos, excess)
-    }
-
     fn calculate_cs_result(
         mut selected_utxos: Vec<OutputGroup>,
         mut required_utxos: Vec<OutputGroup>,
@@ -713,6 +673,58 @@ impl BranchAndBoundCoinSelection {
     }
 }
 
+// Pull UTXOs at random until we have enough to meet the target
+pub(crate) fn single_random_draw(
+    required_utxos: Vec<WeightedUtxo>,
+    optional_utxos: Vec<WeightedUtxo>,
+    target_amount: u64,
+    drain_script: &Script,
+    fee_rate: FeeRate,
+    rng: &mut impl RngCore,
+) -> CoinSelectionResult {
+    let target_amount = target_amount
+        .try_into()
+        .expect("Bitcoin amount to fit into i64");
+
+    let required_utxos: Vec<OutputGroup> = required_utxos
+        .into_iter()
+        .map(|u| OutputGroup::new(u, fee_rate))
+        .collect();
+
+    let mut optional_utxos: Vec<OutputGroup> = optional_utxos
+        .into_iter()
+        .map(|u| OutputGroup::new(u, fee_rate))
+        .collect();
+
+    let curr_value = required_utxos
+        .iter()
+        .fold(0, |acc, x| acc + x.effective_value);
+
+    shuffle_slice(&mut optional_utxos, rng);
+
+    let selected_utxos =
+        optional_utxos
+            .into_iter()
+            .fold((curr_value, vec![]), |(mut amount, mut utxos), utxo| {
+                if amount >= target_amount {
+                    (amount, utxos)
+                } else {
+                    amount += utxo.effective_value;
+                    utxos.push(utxo);
+                    (amount, utxos)
+                }
+            });
+
+    // remaining_amount can't be negative as that would mean the
+    // selection wasn't successful
+    // target_amount = amount_needed + (fee_amount - vin_fees)
+    let remaining_amount = (selected_utxos.0 - target_amount) as u64;
+
+    let excess = decide_change(remaining_amount, fee_rate, drain_script);
+
+    BranchAndBoundCoinSelection::calculate_cs_result(selected_utxos.1, required_utxos, excess)
+}
+
 /// Remove duplicate UTXOs.
 ///
 /// If a UTXO appears in both `required` and `optional`, the appearance in `required` is kept.
@@ -736,22 +748,21 @@ where
 mod test {
     use assert_matches::assert_matches;
     use core::str::FromStr;
+    use rand::rngs::StdRng;
 
     use bdk_chain::ConfirmationTime;
-    use bitcoin::{OutPoint, ScriptBuf, TxOut};
+    use bitcoin::{Amount, ScriptBuf, TxIn, TxOut};
 
     use super::*;
     use crate::types::*;
     use crate::wallet::coin_selection::filter_duplicates;
-    use crate::wallet::Vbytes;
 
-    use rand::rngs::StdRng;
-    use rand::seq::SliceRandom;
+    use rand::prelude::SliceRandom;
     use rand::{Rng, RngCore, SeedableRng};
 
-    // n. of items on witness (1WU) + signature len (1WU) + signature and sighash (72WU)
-    // + pubkey len (1WU) + pubkey (33WU) + script sig len (1 byte, 4WU)
-    const P2WPKH_SATISFACTION_SIZE: usize = 1 + 1 + 72 + 1 + 33 + 4;
+    // signature len (1WU) + signature and sighash (72WU)
+    // + pubkey len (1WU) + pubkey (33WU)
+    const P2WPKH_SATISFACTION_SIZE: usize = 1 + 72 + 1 + 33;
 
     const FEE_AMOUNT: u64 = 50;
 
@@ -763,11 +774,11 @@ mod test {
         ))
         .unwrap();
         WeightedUtxo {
-            satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
+            satisfaction_weight: Weight::from_wu_usize(P2WPKH_SATISFACTION_SIZE),
             utxo: Utxo::Local(LocalOutput {
                 outpoint,
                 txout: TxOut {
-                    value,
+                    value: Amount::from_sat(value),
                     script_pubkey: ScriptBuf::new(),
                 },
                 keychain: KeychainKind::External,
@@ -823,7 +834,7 @@ mod test {
         let mut res = Vec::new();
         for i in 0..utxos_number {
             res.push(WeightedUtxo {
-                satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
+                satisfaction_weight: Weight::from_wu_usize(P2WPKH_SATISFACTION_SIZE),
                 utxo: Utxo::Local(LocalOutput {
                     outpoint: OutPoint::from_str(&format!(
                         "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
@@ -831,7 +842,7 @@ mod test {
                     ))
                     .unwrap(),
                     txout: TxOut {
-                        value: rng.gen_range(0..200000000),
+                        value: Amount::from_sat(rng.gen_range(0..200000000)),
                         script_pubkey: ScriptBuf::new(),
                     },
                     keychain: KeychainKind::External,
@@ -854,7 +865,7 @@ mod test {
     fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<WeightedUtxo> {
         (0..utxos_number)
             .map(|i| WeightedUtxo {
-                satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
+                satisfaction_weight: Weight::from_wu_usize(P2WPKH_SATISFACTION_SIZE),
                 utxo: Utxo::Local(LocalOutput {
                     outpoint: OutPoint::from_str(&format!(
                         "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
@@ -862,7 +873,7 @@ mod test {
                     ))
                     .unwrap(),
                     txout: TxOut {
-                        value: utxos_value,
+                        value: Amount::from_sat(utxos_value),
                         script_pubkey: ScriptBuf::new(),
                     },
                     keychain: KeychainKind::External,
@@ -879,7 +890,7 @@ mod test {
         utxos.shuffle(&mut rng);
         utxos[..utxos_picked_len]
             .iter()
-            .map(|u| u.utxo.txout().value)
+            .map(|u| u.utxo.txout().value.to_sat())
             .sum()
     }
 
@@ -893,7 +904,7 @@ mod test {
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -914,7 +925,7 @@ mod test {
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -935,7 +946,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -957,7 +968,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -975,7 +986,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -992,7 +1003,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1013,7 +1024,7 @@ mod test {
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1034,7 +1045,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1056,7 +1067,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1068,14 +1079,18 @@ mod test {
     fn test_oldest_first_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_oldest_first_test_utxos();
 
-        let target_amount: u64 = utxos.iter().map(|wu| wu.utxo.txout().value).sum::<u64>() - 50;
+        let target_amount: u64 = utxos
+            .iter()
+            .map(|wu| wu.utxo.txout().value.to_sat())
+            .sum::<u64>()
+            - 50;
         let drain_script = ScriptBuf::default();
 
         OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -1083,20 +1098,19 @@ mod test {
     }
 
     #[test]
+    #[ignore = "SRD fn was moved out of BnB"]
     fn test_bnb_coin_selection_success() {
         // In this case bnb won't find a suitable match and single random draw will
         // select three outputs
         let utxos = generate_same_value_utxos(100_000, 20);
-
         let drain_script = ScriptBuf::default();
-
         let target_amount = 250_000 + FEE_AMOUNT;
 
         let result = BranchAndBoundCoinSelection::default()
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1117,7 +1131,7 @@ mod test {
             .coin_select(
                 utxos.clone(),
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1129,6 +1143,7 @@ mod test {
     }
 
     #[test]
+    #[ignore = "no exact match for bnb, previously fell back to SRD"]
     fn test_bnb_coin_selection_optional_are_enough() {
         let utxos = get_test_utxos();
         let drain_script = ScriptBuf::default();
@@ -1138,7 +1153,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1149,6 +1164,26 @@ mod test {
         assert_eq!(result.fee_amount, 136);
     }
 
+    #[test]
+    fn test_single_random_draw_function_success() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut utxos = generate_random_utxos(&mut rng, 300);
+        let target_amount = sum_random_utxos(&mut rng, &mut utxos) + FEE_AMOUNT;
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(1);
+        let drain_script = ScriptBuf::default();
+        let result = single_random_draw(
+            vec![],
+            utxos,
+            target_amount,
+            &drain_script,
+            fee_rate,
+            &mut rng,
+        );
+        assert!(result.selected_amount() > target_amount);
+        assert_eq!(result.fee_amount, (result.selected.len() * 68) as u64);
+    }
+
     #[test]
     #[ignore]
     fn test_bnb_coin_selection_required_not_enough() {
@@ -1163,9 +1198,9 @@ mod test {
         ));
 
         // Defensive assertions, for sanity and in case someone changes the test utxos vector.
-        let amount: u64 = required.iter().map(|u| u.utxo.txout().value).sum();
+        let amount: u64 = required.iter().map(|u| u.utxo.txout().value.to_sat()).sum();
         assert_eq!(amount, 100_000);
-        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value).sum();
+        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value.to_sat()).sum();
         assert!(amount > 150_000);
         let drain_script = ScriptBuf::default();
 
@@ -1175,7 +1210,7 @@ mod test {
             .coin_select(
                 required,
                 optional,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1197,7 +1232,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1215,7 +1250,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -1227,22 +1262,19 @@ mod test {
         let utxos = get_test_utxos();
         let drain_script = ScriptBuf::default();
         let target_amount = 99932; // first utxo's effective value
+        let feerate = FeeRate::BROADCAST_MIN;
 
         let result = BranchAndBoundCoinSelection::new(0)
-            .coin_select(
-                vec![],
-                utxos,
-                FeeRate::from_sat_per_vb(1.0),
-                target_amount,
-                &drain_script,
-            )
+            .coin_select(vec![], utxos, feerate, target_amount, &drain_script)
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
         assert_eq!(result.selected_amount(), 100_000);
-        let input_size = (TXIN_BASE_WEIGHT + P2WPKH_SATISFACTION_SIZE).vbytes();
+        let input_weight =
+            TxIn::default().segwit_weight().to_wu() + P2WPKH_SATISFACTION_SIZE as u64;
         // the final fee rate should be exactly the same as the fee rate given
-        assert!((1.0 - (result.fee_amount as f32 / input_size as f32)).abs() < f32::EPSILON);
+        let result_feerate = Amount::from_sat(result.fee_amount) / Weight::from_wu(input_weight);
+        assert_eq!(result_feerate, feerate);
     }
 
     #[test]
@@ -1258,7 +1290,7 @@ mod test {
                 .coin_select(
                     vec![],
                     optional_utxos,
-                    FeeRate::from_sat_per_vb(0.0),
+                    FeeRate::ZERO,
                     target_amount,
                     &drain_script,
                 )
@@ -1270,7 +1302,7 @@ mod test {
     #[test]
     #[should_panic(expected = "BnBNoExactMatch")]
     fn test_bnb_function_no_exact_match() {
-        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(10);
         let utxos: Vec<OutputGroup> = get_test_utxos()
             .into_iter()
             .map(|u| OutputGroup::new(u, fee_rate))
@@ -1279,7 +1311,7 @@ mod test {
         let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
 
         let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
@@ -1300,7 +1332,7 @@ mod test {
     #[test]
     #[should_panic(expected = "BnBTotalTriesExceeded")]
     fn test_bnb_function_tries_exceeded() {
-        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(10);
         let utxos: Vec<OutputGroup> = generate_same_value_utxos(100_000, 100_000)
             .into_iter()
             .map(|u| OutputGroup::new(u, fee_rate))
@@ -1309,7 +1341,7 @@ mod test {
         let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
         let target_amount = 20_000 + FEE_AMOUNT;
 
         let drain_script = ScriptBuf::default();
@@ -1331,9 +1363,9 @@ mod test {
     // The match won't be exact but still in the range
     #[test]
     fn test_bnb_function_almost_exact_match_with_fees() {
-        let fee_rate = FeeRate::from_sat_per_vb(1.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(1);
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
 
         let utxos: Vec<_> = generate_same_value_utxos(50_000, 10)
             .into_iter()
@@ -1346,7 +1378,7 @@ mod test {
 
         // 2*(value of 1 utxo)  - 2*(1 utxo fees with 1.0sat/vbyte fee rate) -
         // cost_of_change + 5.
-        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as i64 + 5;
+        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change as i64 + 5;
 
         let drain_script = ScriptBuf::default();
 
@@ -1371,7 +1403,7 @@ mod test {
     fn test_bnb_function_exact_match_more_utxos() {
         let seed = [0; 32];
         let mut rng: StdRng = SeedableRng::from_seed(seed);
-        let fee_rate = FeeRate::from_sat_per_vb(0.0);
+        let fee_rate = FeeRate::ZERO;
 
         for _ in 0..200 {
             let optional_utxos: Vec<_> = generate_random_utxos(&mut rng, 40)
@@ -1397,7 +1429,7 @@ mod test {
                     curr_value,
                     curr_available_value,
                     target_amount,
-                    0.0,
+                    0,
                     &drain_script,
                     fee_rate,
                 )
@@ -1406,34 +1438,6 @@ mod test {
         }
     }
 
-    #[test]
-    fn test_single_random_draw_function_success() {
-        let seed = [0; 32];
-        let mut rng: StdRng = SeedableRng::from_seed(seed);
-        let mut utxos = generate_random_utxos(&mut rng, 300);
-        let target_amount = sum_random_utxos(&mut rng, &mut utxos) + FEE_AMOUNT;
-
-        let fee_rate = FeeRate::from_sat_per_vb(1.0);
-        let utxos: Vec<OutputGroup> = utxos
-            .into_iter()
-            .map(|u| OutputGroup::new(u, fee_rate))
-            .collect();
-
-        let drain_script = ScriptBuf::default();
-
-        let result = BranchAndBoundCoinSelection::default().single_random_draw(
-            vec![],
-            utxos,
-            0,
-            target_amount as i64,
-            &drain_script,
-            fee_rate,
-        );
-
-        assert!(result.selected_amount() > target_amount);
-        assert_eq!(result.fee_amount, (result.selected.len() * 68) as u64);
-    }
-
     #[test]
     fn test_bnb_exclude_negative_effective_value() {
         let utxos = get_test_utxos();
@@ -1442,7 +1446,7 @@ mod test {
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             vec![],
             utxos,
-            FeeRate::from_sat_per_vb(10.0),
+            FeeRate::from_sat_per_vb_unchecked(10),
             500_000,
             &drain_script,
         );
@@ -1461,14 +1465,14 @@ mod test {
         let utxos = get_test_utxos();
         let drain_script = ScriptBuf::default();
 
-        let (required, optional) = utxos
-            .into_iter()
-            .partition(|u| matches!(u, WeightedUtxo { utxo, .. } if utxo.txout().value < 1000));
+        let (required, optional) = utxos.into_iter().partition(
+            |u| matches!(u, WeightedUtxo { utxo, .. } if utxo.txout().value.to_sat() < 1000),
+        );
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             required,
             optional,
-            FeeRate::from_sat_per_vb(10.0),
+            FeeRate::from_sat_per_vb_unchecked(10),
             500_000,
             &drain_script,
         );
@@ -1490,7 +1494,7 @@ mod test {
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             utxos,
             vec![],
-            FeeRate::from_sat_per_vb(10_000.0),
+            FeeRate::from_sat_per_vb_unchecked(10_000),
             500_000,
             &drain_script,
         );
@@ -1508,11 +1512,11 @@ mod test {
     fn test_filter_duplicates() {
         fn utxo(txid: &str, value: u64) -> WeightedUtxo {
             WeightedUtxo {
-                satisfaction_weight: 0,
+                satisfaction_weight: Weight::ZERO,
                 utxo: Utxo::Local(LocalOutput {
                     outpoint: OutPoint::new(bitcoin::hashes::Hash::hash(txid.as_bytes()), 0),
                     txout: TxOut {
-                        value,
+                        value: Amount::from_sat(value),
                         script_pubkey: ScriptBuf::new(),
                     },
                     keychain: KeychainKind::External,

crates/wallet/src/wallet/error.rs

@@ -14,9 +14,9 @@
 use crate::descriptor::policy::PolicyError;
 use crate::descriptor::DescriptorError;
 use crate::wallet::coin_selection;
-use crate::{descriptor, FeeRate, KeychainKind};
+use crate::{descriptor, KeychainKind};
 use alloc::string::String;
-use bitcoin::{absolute, psbt, OutPoint, Sequence, Txid};
+use bitcoin::{absolute, psbt, Amount, OutPoint, Sequence, Txid};
 use core::fmt;
 
 /// Errors returned by miniscript when updating inconsistent PSBTs
@@ -47,11 +47,9 @@ impl std::error::Error for MiniscriptPsbtError {}
 /// Error returned from [`TxBuilder::finish`]
 ///
 /// [`TxBuilder::finish`]: crate::wallet::tx_builder::TxBuilder::finish
-pub enum CreateTxError<P> {
+pub enum CreateTxError {
     /// There was a problem with the descriptors passed in
     Descriptor(DescriptorError),
-    /// We were unable to write wallet data to the persistence backend
-    Persist(P),
     /// There was a problem while extracting and manipulating policies
     Policy(PolicyError),
     /// Spending policy is not compatible with this [`KeychainKind`]
@@ -78,29 +76,20 @@ pub enum CreateTxError<P> {
     },
     /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
     FeeTooLow {
-        /// Required fee absolute value (satoshi)
-        required: u64,
+        /// Required fee absolute value [`Amount`]
+        required: Amount,
     },
     /// When bumping a tx the fee rate requested is lower than required
     FeeRateTooLow {
-        /// Required fee rate (satoshi/vbyte)
-        required: FeeRate,
+        /// Required fee rate
+        required: bitcoin::FeeRate,
     },
     /// `manually_selected_only` option is selected but no utxo has been passed
     NoUtxosSelected,
     /// Output created is under the dust limit, 546 satoshis
     OutputBelowDustLimit(usize),
-    /// The `change_policy` was set but the wallet does not have a change_descriptor
-    ChangePolicyDescriptor,
     /// There was an error with coin selection
     CoinSelection(coin_selection::Error),
-    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
-    InsufficientFunds {
-        /// Sats needed for some transaction
-        needed: u64,
-        /// Sats available for spending
-        available: u64,
-    },
     /// Cannot build a tx without recipients
     NoRecipients,
     /// Partially signed bitcoin transaction error
@@ -119,20 +108,10 @@ pub enum CreateTxError<P> {
     MiniscriptPsbt(MiniscriptPsbtError),
 }
 
-impl<P> fmt::Display for CreateTxError<P>
-where
-    P: fmt::Display,
-{
+impl fmt::Display for CreateTxError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::Descriptor(e) => e.fmt(f),
-            Self::Persist(e) => {
-                write!(
-                    f,
-                    "failed to write wallet data to persistence backend: {}",
-                    e
-                )
-            }
             Self::Policy(e) => e.fmt(f),
             CreateTxError::SpendingPolicyRequired(keychain_kind) => {
                 write!(f, "Spending policy required: {:?}", keychain_kind)
@@ -163,13 +142,15 @@ where
                 )
             }
             CreateTxError::FeeTooLow { required } => {
-                write!(f, "Fee to low: required {} sat", required)
+                write!(f, "Fee to low: required {}", required.display_dynamic())
             }
             CreateTxError::FeeRateTooLow { required } => {
                 write!(
                     f,
-                    "Fee rate too low: required {} sat/vbyte",
-                    required.as_sat_per_vb()
+                    // Note: alternate fmt as sat/vb (ceil) available in bitcoin-0.31
+                    //"Fee rate too low: required {required:#}"
+                    "Fee rate too low: required {} sat/vb",
+                    crate::floating_rate!(required)
                 )
             }
             CreateTxError::NoUtxosSelected => {
@@ -178,20 +159,7 @@ where
             CreateTxError::OutputBelowDustLimit(limit) => {
                 write!(f, "Output below the dust limit: {}", limit)
             }
-            CreateTxError::ChangePolicyDescriptor => {
-                write!(
-                    f,
-                    "The `change_policy` can be set only if the wallet has a change_descriptor"
-                )
-            }
             CreateTxError::CoinSelection(e) => e.fmt(f),
-            CreateTxError::InsufficientFunds { needed, available } => {
-                write!(
-                    f,
-                    "Insufficient funds: {} sat available of {} sat needed",
-                    available, needed
-                )
-            }
             CreateTxError::NoRecipients => {
                 write!(f, "Cannot build tx without recipients")
             }
@@ -212,38 +180,38 @@ where
     }
 }
 
-impl<P> From<descriptor::error::Error> for CreateTxError<P> {
+impl From<descriptor::error::Error> for CreateTxError {
     fn from(err: descriptor::error::Error) -> Self {
         CreateTxError::Descriptor(err)
     }
 }
 
-impl<P> From<PolicyError> for CreateTxError<P> {
+impl From<PolicyError> for CreateTxError {
     fn from(err: PolicyError) -> Self {
         CreateTxError::Policy(err)
     }
 }
 
-impl<P> From<MiniscriptPsbtError> for CreateTxError<P> {
+impl From<MiniscriptPsbtError> for CreateTxError {
     fn from(err: MiniscriptPsbtError) -> Self {
         CreateTxError::MiniscriptPsbt(err)
     }
 }
 
-impl<P> From<psbt::Error> for CreateTxError<P> {
+impl From<psbt::Error> for CreateTxError {
     fn from(err: psbt::Error) -> Self {
         CreateTxError::Psbt(err)
     }
 }
 
-impl<P> From<coin_selection::Error> for CreateTxError<P> {
+impl From<coin_selection::Error> for CreateTxError {
     fn from(err: coin_selection::Error) -> Self {
         CreateTxError::CoinSelection(err)
     }
 }
 
 #[cfg(feature = "std")]
-impl<P: core::fmt::Display + core::fmt::Debug> std::error::Error for CreateTxError<P> {}
+impl std::error::Error for CreateTxError {}
 
 #[derive(Debug)]
 /// Error returned from [`Wallet::build_fee_bump`]

crates/wallet/src/wallet/export.rs

@@ -20,8 +20,8 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::export::*;
-//! # use bdk::*;
+//! # use bdk_wallet::export::*;
+//! # use bdk_wallet::*;
 //! let import = r#"{
 //!     "descriptor": "wpkh([c258d2e4\/84h\/1h\/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe\/0\/*)",
 //!     "blockheight":1782088,
@@ -29,33 +29,35 @@
 //! }"#;
 //!
 //! let import = FullyNodedExport::from_str(import)?;
-//! let wallet = Wallet::new_no_persist(
-//!     &import.descriptor(),
-//!     import.change_descriptor().as_ref(),
-//!     Network::Testnet,
-//! )?;
+//! let wallet = Wallet::create(
+//!     import.descriptor(),
+//!     import.change_descriptor().expect("change descriptor"),
+//! )
+//! .network(Network::Testnet)
+//! .create_wallet_no_persist()?;
 //! # Ok::<_, Box<dyn std::error::Error>>(())
 //! ```
 //!
 //! ### Export a `Wallet`
 //! ```
 //! # use bitcoin::*;
-//! # use bdk::wallet::export::*;
-//! # use bdk::*;
-//! let wallet = Wallet::new_no_persist(
+//! # use bdk_wallet::export::*;
+//! # use bdk_wallet::*;
+//! let wallet = Wallet::create(
 //!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/0/*)",
-//!     Some("wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)"),
-//!     Network::Testnet,
-//! )?;
+//!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)",
+//! )
+//! .network(Network::Testnet)
+//! .create_wallet_no_persist()?;
 //! let export = FullyNodedExport::export_wallet(&wallet, "exported wallet", true).unwrap();
 //!
 //! println!("Exported: {}", export.to_string());
 //! # Ok::<_, Box<dyn std::error::Error>>(())
 //! ```
 
+use alloc::string::String;
+use core::fmt;
 use core::str::FromStr;
-
-use alloc::string::{String, ToString};
 use serde::{Deserialize, Serialize};
 
 use miniscript::descriptor::{ShInner, WshInner};
@@ -80,9 +82,9 @@ pub struct FullyNodedExport {
     pub label: String,
 }
 
-impl ToString for FullyNodedExport {
-    fn to_string(&self) -> String {
-        serde_json::to_string(self).unwrap()
+impl fmt::Display for FullyNodedExport {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", serde_json::to_string(self).unwrap())
     }
 }
 
@@ -110,13 +112,13 @@ impl FullyNodedExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet<D>(
-        wallet: &Wallet<D>,
+    pub fn export_wallet(
+        wallet: &Wallet,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
         let descriptor = wallet
-            .get_descriptor_for_keychain(KeychainKind::External)
+            .public_descriptor(KeychainKind::External)
             .to_string_with_secret(
                 &wallet
                     .get_signers(KeychainKind::External)
@@ -128,7 +130,7 @@ impl FullyNodedExport {
         let blockheight = if include_blockheight {
             wallet.transactions().next().map_or(0, |canonical_tx| {
                 match canonical_tx.chain_position {
-                    bdk_chain::ChainPosition::Confirmed(a) => a.confirmation_height,
+                    bdk_chain::ChainPosition::Confirmed(a) => a.block_id.height,
                     bdk_chain::ChainPosition::Unconfirmed(_) => 0,
                 }
             })
@@ -142,19 +144,17 @@ impl FullyNodedExport {
             blockheight,
         };
 
-        let change_descriptor = match wallet.public_descriptor(KeychainKind::Internal).is_some() {
-            false => None,
-            true => {
-                let descriptor = wallet
-                    .get_descriptor_for_keychain(KeychainKind::Internal)
-                    .to_string_with_secret(
-                        &wallet
-                            .get_signers(KeychainKind::Internal)
-                            .as_key_map(wallet.secp_ctx()),
-                    );
-                Some(remove_checksum(descriptor))
-            }
+        let change_descriptor = {
+            let descriptor = wallet
+                .public_descriptor(KeychainKind::Internal)
+                .to_string_with_secret(
+                    &wallet
+                        .get_signers(KeychainKind::Internal)
+                        .as_key_map(wallet.secp_ctx()),
+                );
+            Some(remove_checksum(descriptor))
         };
+
         if export.change_descriptor() != change_descriptor {
             return Err("Incompatible change descriptor");
         }
@@ -166,7 +166,7 @@ impl FullyNodedExport {
         fn check_ms<Ctx: ScriptContext>(
             terminal: &Terminal<String, Ctx>,
         ) -> Result<(), &'static str> {
-            if let Terminal::Multi(_, _) = terminal {
+            if let Terminal::Multi(_) = terminal {
                 Ok(())
             } else {
                 Err("The descriptor contains operators not supported by Bitcoin Core")
@@ -189,6 +189,7 @@ impl FullyNodedExport {
                 WshInner::SortedMulti(_) => Ok(()),
                 WshInner::Ms(ms) => check_ms(&ms.node),
             },
+            Descriptor::Tr(_) => Ok(()),
             _ => Err("The descriptor is not compatible with Bitcoin Core"),
         }
     }
@@ -214,39 +215,51 @@ impl FullyNodedExport {
 mod test {
     use core::str::FromStr;
 
-    use bdk_chain::{BlockId, ConfirmationTime};
+    use crate::std::string::ToString;
+    use bdk_chain::{BlockId, ConfirmationBlockTime};
     use bitcoin::hashes::Hash;
-    use bitcoin::{BlockHash, Network, Transaction};
+    use bitcoin::{transaction, BlockHash, Network, Transaction};
 
     use super::*;
-    use crate::wallet::Wallet;
+    use crate::Wallet;
 
-    fn get_test_wallet(
-        descriptor: &str,
-        change_descriptor: Option<&str>,
-        network: Network,
-    ) -> Wallet<()> {
-        let mut wallet = Wallet::new_no_persist(descriptor, change_descriptor, network).unwrap();
+    fn get_test_wallet(descriptor: &str, change_descriptor: &str, network: Network) -> Wallet {
+        use crate::wallet::Update;
+        use bdk_chain::TxGraph;
+        let mut wallet = Wallet::create(descriptor.to_string(), change_descriptor.to_string())
+            .network(network)
+            .create_wallet_no_persist()
+            .expect("must create wallet");
         let transaction = Transaction {
             input: vec![],
             output: vec![],
-            version: 0,
+            version: transaction::Version::non_standard(0),
             lock_time: bitcoin::absolute::LockTime::ZERO,
         };
+        let txid = transaction.compute_txid();
+        let block_id = BlockId {
+            height: 5000,
+            hash: BlockHash::all_zeros(),
+        };
+        wallet.insert_checkpoint(block_id).unwrap();
         wallet
             .insert_checkpoint(BlockId {
                 height: 5001,
                 hash: BlockHash::all_zeros(),
             })
             .unwrap();
+        wallet.insert_tx(transaction);
+        let anchor = ConfirmationBlockTime {
+            confirmation_time: 0,
+            block_id,
+        };
+        let mut graph = TxGraph::default();
+        let _ = graph.insert_anchor(txid, anchor);
         wallet
-            .insert_tx(
-                transaction,
-                ConfirmationTime::Confirmed {
-                    height: 5000,
-                    time: 0,
-                },
-            )
+            .apply_update(Update {
+                graph,
+                ..Default::default()
+            })
             .unwrap();
         wallet
     }
@@ -256,7 +269,7 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.descriptor(), descriptor);
@@ -268,13 +281,14 @@ mod test {
     #[test]
     #[should_panic(expected = "Incompatible change descriptor")]
     fn test_export_no_change() {
-        // This wallet explicitly doesn't have a change descriptor. It should be impossible to
+        // The wallet's change descriptor has no wildcard. It should be impossible to
         // export, because exporting this kind of external descriptor normally implies the
-        // existence of an internal descriptor
+        // existence of a compatible internal descriptor
 
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/0)";
 
-        let wallet = get_test_wallet(descriptor, None, Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -287,7 +301,7 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/50'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -304,7 +318,7 @@ mod test {
                                        [c98b1535/48'/0'/0'/2']tpubDCDi5W4sP6zSnzJeowy8rQDVhBdRARaPhK1axABi8V1661wEPeanpEXj4ZLAUEoikVtoWcyK26TKKJSecSfeKxwHCcRrge9k1ybuiL71z4a/1/*\
                                  ))";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Testnet);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Testnet);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.descriptor(), descriptor);
@@ -313,12 +327,24 @@ mod test {
         assert_eq!(export.label, "Test Label");
     }
 
+    #[test]
+    fn test_export_tr() {
+        let descriptor = "tr([73c5da0a/86'/0'/0']tprv8fMn4hSKPRC1oaCPqxDb1JWtgkpeiQvZhsr8W2xuy3GEMkzoArcAWTfJxYb6Wj8XNNDWEjfYKK4wGQXh3ZUXhDF2NcnsALpWTeSwarJt7Vc/0/*)";
+        let change_descriptor = "tr([73c5da0a/86'/0'/0']tprv8fMn4hSKPRC1oaCPqxDb1JWtgkpeiQvZhsr8W2xuy3GEMkzoArcAWTfJxYb6Wj8XNNDWEjfYKK4wGQXh3ZUXhDF2NcnsALpWTeSwarJt7Vc/1/*)";
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Testnet);
+        let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
+        assert_eq!(export.descriptor(), descriptor);
+        assert_eq!(export.change_descriptor(), Some(change_descriptor.into()));
+        assert_eq!(export.blockheight, 5000);
+        assert_eq!(export.label, "Test Label");
+    }
+
     #[test]
     fn test_export_to_json() {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.to_string(), "{\"descriptor\":\"wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44\'/0\'/0\'/0/*)\",\"blockheight\":5000,\"label\":\"Test Label\"}");

crates/wallet/src/wallet/hardwaresigner.rs

@@ -14,11 +14,11 @@
 //! This module contains HWISigner, an implementation of a [TransactionSigner] to be
 //! used with hardware wallets.
 //! ```no_run
-//! # use bdk::bitcoin::Network;
-//! # use bdk::signer::SignerOrdering;
-//! # use bdk::wallet::hardwaresigner::HWISigner;
-//! # use bdk::wallet::AddressIndex::New;
-//! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
+//! # use bdk_wallet::bitcoin::Network;
+//! # use bdk_wallet::signer::SignerOrdering;
+//! # use bdk_wallet::hardwaresigner::HWISigner;
+//! # use bdk_wallet::AddressIndex::New;
+//! # use bdk_wallet::{CreateParams, KeychainKind, SignOptions};
 //! # use hwi::HWIClient;
 //! # use std::sync::Arc;
 //! #
@@ -30,11 +30,7 @@
 //! let first_device = devices.remove(0)?;
 //! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
 //!
-//! # let mut wallet = Wallet::new_no_persist(
-//! #     "",
-//! #     None,
-//! #     Network::Testnet,
-//! # )?;
+//! # let mut wallet = CreateParams::new("", "", Network::Testnet)?.create_wallet_no_persist()?;
 //! #
 //! // Adding the hardware signer to the BDK wallet
 //! wallet.add_signer(
@@ -48,8 +44,8 @@
 //! ```
 
 use bitcoin::bip32::Fingerprint;
-use bitcoin::psbt::PartiallySignedTransaction;
 use bitcoin::secp256k1::{All, Secp256k1};
+use bitcoin::Psbt;
 
 use hwi::error::Error;
 use hwi::types::{HWIChain, HWIDevice};
@@ -87,7 +83,7 @@ impl SignerCommon for HWISigner {
 impl TransactionSigner for HWISigner {
     fn sign_transaction(
         &self,
-        psbt: &mut PartiallySignedTransaction,
+        psbt: &mut Psbt,
         _sign_options: &crate::SignOptions,
         _secp: &crate::wallet::utils::SecpCtx,
     ) -> Result<(), SignerError> {

crates/wallet/src/wallet/params.rs

@@ -0,0 +1,213 @@
+use alloc::boxed::Box;
+use bdk_chain::{keychain_txout::DEFAULT_LOOKAHEAD, PersistAsyncWith, PersistWith};
+use bitcoin::{BlockHash, Network};
+use miniscript::descriptor::KeyMap;
+
+use crate::{
+    descriptor::{DescriptorError, ExtendedDescriptor, IntoWalletDescriptor},
+    utils::SecpCtx,
+    KeychainKind, Wallet,
+};
+
+use super::{ChangeSet, LoadError, PersistedWallet};
+
+/// This atrocity is to avoid having type parameters on [`CreateParams`] and [`LoadParams`].
+///
+/// The better option would be to do `Box<dyn IntoWalletDescriptor>`, but we cannot due to Rust's
+/// [object safety rules](https://doc.rust-lang.org/reference/items/traits.html#object-safety).
+type DescriptorToExtract = Box<
+    dyn FnOnce(&SecpCtx, Network) -> Result<(ExtendedDescriptor, KeyMap), DescriptorError>
+        + 'static,
+>;
+
+fn make_descriptor_to_extract<D>(descriptor: D) -> DescriptorToExtract
+where
+    D: IntoWalletDescriptor + 'static,
+{
+    Box::new(|secp, network| descriptor.into_wallet_descriptor(secp, network))
+}
+
+/// Parameters for [`Wallet::create`] or [`PersistedWallet::create`].
+#[must_use]
+pub struct CreateParams {
+    pub(crate) descriptor: DescriptorToExtract,
+    pub(crate) descriptor_keymap: KeyMap,
+    pub(crate) change_descriptor: DescriptorToExtract,
+    pub(crate) change_descriptor_keymap: KeyMap,
+    pub(crate) network: Network,
+    pub(crate) genesis_hash: Option<BlockHash>,
+    pub(crate) lookahead: u32,
+}
+
+impl CreateParams {
+    /// Construct parameters with provided `descriptor`, `change_descriptor` and `network`.
+    ///
+    /// Default values: `genesis_hash` = `None`, `lookahead` = [`DEFAULT_LOOKAHEAD`]
+    pub fn new<D: IntoWalletDescriptor + 'static>(descriptor: D, change_descriptor: D) -> Self {
+        Self {
+            descriptor: make_descriptor_to_extract(descriptor),
+            descriptor_keymap: KeyMap::default(),
+            change_descriptor: make_descriptor_to_extract(change_descriptor),
+            change_descriptor_keymap: KeyMap::default(),
+            network: Network::Bitcoin,
+            genesis_hash: None,
+            lookahead: DEFAULT_LOOKAHEAD,
+        }
+    }
+
+    /// Extend the given `keychain`'s `keymap`.
+    pub fn keymap(mut self, keychain: KeychainKind, keymap: KeyMap) -> Self {
+        match keychain {
+            KeychainKind::External => &mut self.descriptor_keymap,
+            KeychainKind::Internal => &mut self.change_descriptor_keymap,
+        }
+        .extend(keymap);
+        self
+    }
+
+    /// Set `network`.
+    pub fn network(mut self, network: Network) -> Self {
+        self.network = network;
+        self
+    }
+
+    /// Use a custom `genesis_hash`.
+    pub fn genesis_hash(mut self, genesis_hash: BlockHash) -> Self {
+        self.genesis_hash = Some(genesis_hash);
+        self
+    }
+
+    /// Use custom lookahead value.
+    pub fn lookahead(mut self, lookahead: u32) -> Self {
+        self.lookahead = lookahead;
+        self
+    }
+
+    /// Create [`PersistedWallet`] with the given `Db`.
+    pub fn create_wallet<Db>(
+        self,
+        db: &mut Db,
+    ) -> Result<PersistedWallet, <Wallet as PersistWith<Db>>::CreateError>
+    where
+        Wallet: PersistWith<Db, CreateParams = Self>,
+    {
+        PersistedWallet::create(db, self)
+    }
+
+    /// Create [`PersistedWallet`] with the given async `Db`.
+    pub async fn create_wallet_async<Db>(
+        self,
+        db: &mut Db,
+    ) -> Result<PersistedWallet, <Wallet as PersistAsyncWith<Db>>::CreateError>
+    where
+        Wallet: PersistAsyncWith<Db, CreateParams = Self>,
+    {
+        PersistedWallet::create_async(db, self).await
+    }
+
+    /// Create [`Wallet`] without persistence.
+    pub fn create_wallet_no_persist(self) -> Result<Wallet, DescriptorError> {
+        Wallet::create_with_params(self)
+    }
+}
+
+/// Parameters for [`Wallet::load`] or [`PersistedWallet::load`].
+#[must_use]
+pub struct LoadParams {
+    pub(crate) descriptor_keymap: KeyMap,
+    pub(crate) change_descriptor_keymap: KeyMap,
+    pub(crate) lookahead: u32,
+    pub(crate) check_network: Option<Network>,
+    pub(crate) check_genesis_hash: Option<BlockHash>,
+    pub(crate) check_descriptor: Option<DescriptorToExtract>,
+    pub(crate) check_change_descriptor: Option<DescriptorToExtract>,
+}
+
+impl LoadParams {
+    /// Construct parameters with default values.
+    ///
+    /// Default values: `lookahead` = [`DEFAULT_LOOKAHEAD`]
+    pub fn new() -> Self {
+        Self {
+            descriptor_keymap: KeyMap::default(),
+            change_descriptor_keymap: KeyMap::default(),
+            lookahead: DEFAULT_LOOKAHEAD,
+            check_network: None,
+            check_genesis_hash: None,
+            check_descriptor: None,
+            check_change_descriptor: None,
+        }
+    }
+
+    /// Extend the given `keychain`'s `keymap`.
+    pub fn keymap(mut self, keychain: KeychainKind, keymap: KeyMap) -> Self {
+        match keychain {
+            KeychainKind::External => &mut self.descriptor_keymap,
+            KeychainKind::Internal => &mut self.change_descriptor_keymap,
+        }
+        .extend(keymap);
+        self
+    }
+
+    /// Checks that `descriptor` of `keychain` matches this, and extracts private keys (if
+    /// avaliable).
+    pub fn descriptors<D>(mut self, descriptor: D, change_descriptor: D) -> Self
+    where
+        D: IntoWalletDescriptor + 'static,
+    {
+        self.check_descriptor = Some(make_descriptor_to_extract(descriptor));
+        self.check_change_descriptor = Some(make_descriptor_to_extract(change_descriptor));
+        self
+    }
+
+    /// Check for `network`.
+    pub fn network(mut self, network: Network) -> Self {
+        self.check_network = Some(network);
+        self
+    }
+
+    /// Check for a `genesis_hash`.
+    pub fn genesis_hash(mut self, genesis_hash: BlockHash) -> Self {
+        self.check_genesis_hash = Some(genesis_hash);
+        self
+    }
+
+    /// Use custom lookahead value.
+    pub fn lookahead(mut self, lookahead: u32) -> Self {
+        self.lookahead = lookahead;
+        self
+    }
+
+    /// Load [`PersistedWallet`] with the given `Db`.
+    pub fn load_wallet<Db>(
+        self,
+        db: &mut Db,
+    ) -> Result<Option<PersistedWallet>, <Wallet as PersistWith<Db>>::LoadError>
+    where
+        Wallet: PersistWith<Db, LoadParams = Self>,
+    {
+        PersistedWallet::load(db, self)
+    }
+
+    /// Load [`PersistedWallet`] with the given async `Db`.
+    pub async fn load_wallet_async<Db>(
+        self,
+        db: &mut Db,
+    ) -> Result<Option<PersistedWallet>, <Wallet as PersistAsyncWith<Db>>::LoadError>
+    where
+        Wallet: PersistAsyncWith<Db, LoadParams = Self>,
+    {
+        PersistedWallet::load_async(db, self).await
+    }
+
+    /// Load [`Wallet`] without persistence.
+    pub fn load_wallet_no_persist(self, changeset: ChangeSet) -> Result<Option<Wallet>, LoadError> {
+        Wallet::load_with_params(changeset, self)
+    }
+}
+
+impl Default for LoadParams {
+    fn default() -> Self {
+        Self::new()
+    }
+}

crates/wallet/src/wallet/persisted.rs

@@ -0,0 +1,171 @@
+use core::fmt;
+
+use crate::{descriptor::DescriptorError, Wallet};
+
+/// Represents a persisted wallet.
+pub type PersistedWallet = bdk_chain::Persisted<Wallet>;
+
+#[cfg(feature = "rusqlite")]
+impl<'c> chain::PersistWith<bdk_chain::rusqlite::Transaction<'c>> for Wallet {
+    type CreateParams = crate::CreateParams;
+    type LoadParams = crate::LoadParams;
+
+    type CreateError = CreateWithPersistError<bdk_chain::rusqlite::Error>;
+    type LoadError = LoadWithPersistError<bdk_chain::rusqlite::Error>;
+    type PersistError = bdk_chain::rusqlite::Error;
+
+    fn create(
+        db: &mut bdk_chain::rusqlite::Transaction<'c>,
+        params: Self::CreateParams,
+    ) -> Result<Self, Self::CreateError> {
+        let mut wallet =
+            Self::create_with_params(params).map_err(CreateWithPersistError::Descriptor)?;
+        if let Some(changeset) = wallet.take_staged() {
+            changeset
+                .persist_to_sqlite(db)
+                .map_err(CreateWithPersistError::Persist)?;
+        }
+        Ok(wallet)
+    }
+
+    fn load(
+        conn: &mut bdk_chain::rusqlite::Transaction<'c>,
+        params: Self::LoadParams,
+    ) -> Result<Option<Self>, Self::LoadError> {
+        let changeset =
+            crate::ChangeSet::from_sqlite(conn).map_err(LoadWithPersistError::Persist)?;
+        if chain::Merge::is_empty(&changeset) {
+            return Ok(None);
+        }
+        Self::load_with_params(changeset, params).map_err(LoadWithPersistError::InvalidChangeSet)
+    }
+
+    fn persist(
+        db: &mut bdk_chain::rusqlite::Transaction<'c>,
+        changeset: &<Self as chain::Staged>::ChangeSet,
+    ) -> Result<(), Self::PersistError> {
+        changeset.persist_to_sqlite(db)
+    }
+}
+
+#[cfg(feature = "rusqlite")]
+impl chain::PersistWith<bdk_chain::rusqlite::Connection> for Wallet {
+    type CreateParams = crate::CreateParams;
+    type LoadParams = crate::LoadParams;
+
+    type CreateError = CreateWithPersistError<bdk_chain::rusqlite::Error>;
+    type LoadError = LoadWithPersistError<bdk_chain::rusqlite::Error>;
+    type PersistError = bdk_chain::rusqlite::Error;
+
+    fn create(
+        db: &mut bdk_chain::rusqlite::Connection,
+        params: Self::CreateParams,
+    ) -> Result<Self, Self::CreateError> {
+        let mut db_tx = db.transaction().map_err(CreateWithPersistError::Persist)?;
+        let wallet = chain::PersistWith::create(&mut db_tx, params)?;
+        db_tx.commit().map_err(CreateWithPersistError::Persist)?;
+        Ok(wallet)
+    }
+
+    fn load(
+        db: &mut bdk_chain::rusqlite::Connection,
+        params: Self::LoadParams,
+    ) -> Result<Option<Self>, Self::LoadError> {
+        let mut db_tx = db.transaction().map_err(LoadWithPersistError::Persist)?;
+        let wallet_opt = chain::PersistWith::load(&mut db_tx, params)?;
+        db_tx.commit().map_err(LoadWithPersistError::Persist)?;
+        Ok(wallet_opt)
+    }
+
+    fn persist(
+        db: &mut bdk_chain::rusqlite::Connection,
+        changeset: &<Self as chain::Staged>::ChangeSet,
+    ) -> Result<(), Self::PersistError> {
+        let db_tx = db.transaction()?;
+        changeset.persist_to_sqlite(&db_tx)?;
+        db_tx.commit()
+    }
+}
+
+#[cfg(feature = "file_store")]
+impl chain::PersistWith<bdk_file_store::Store<crate::ChangeSet>> for Wallet {
+    type CreateParams = crate::CreateParams;
+    type LoadParams = crate::LoadParams;
+    type CreateError = CreateWithPersistError<std::io::Error>;
+    type LoadError =
+        LoadWithPersistError<bdk_file_store::AggregateChangesetsError<crate::ChangeSet>>;
+    type PersistError = std::io::Error;
+
+    fn create(
+        db: &mut bdk_file_store::Store<crate::ChangeSet>,
+        params: Self::CreateParams,
+    ) -> Result<Self, Self::CreateError> {
+        let mut wallet =
+            Self::create_with_params(params).map_err(CreateWithPersistError::Descriptor)?;
+        if let Some(changeset) = wallet.take_staged() {
+            db.append_changeset(&changeset)
+                .map_err(CreateWithPersistError::Persist)?;
+        }
+        Ok(wallet)
+    }
+
+    fn load(
+        db: &mut bdk_file_store::Store<crate::ChangeSet>,
+        params: Self::LoadParams,
+    ) -> Result<Option<Self>, Self::LoadError> {
+        let changeset = db
+            .aggregate_changesets()
+            .map_err(LoadWithPersistError::Persist)?
+            .unwrap_or_default();
+        Self::load_with_params(changeset, params).map_err(LoadWithPersistError::InvalidChangeSet)
+    }
+
+    fn persist(
+        db: &mut bdk_file_store::Store<crate::ChangeSet>,
+        changeset: &<Self as chain::Staged>::ChangeSet,
+    ) -> Result<(), Self::PersistError> {
+        db.append_changeset(changeset)
+    }
+}
+
+/// Error type for [`PersistedWallet::load`].
+#[derive(Debug, PartialEq)]
+pub enum LoadWithPersistError<E> {
+    /// Error from persistence.
+    Persist(E),
+    /// Occurs when the loaded changeset cannot construct [`Wallet`].
+    InvalidChangeSet(crate::LoadError),
+}
+
+impl<E: fmt::Display> fmt::Display for LoadWithPersistError<E> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Persist(err) => fmt::Display::fmt(err, f),
+            Self::InvalidChangeSet(err) => fmt::Display::fmt(&err, f),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl<E: fmt::Debug + fmt::Display> std::error::Error for LoadWithPersistError<E> {}
+
+/// Error type for [`PersistedWallet::create`].
+#[derive(Debug)]
+pub enum CreateWithPersistError<E> {
+    /// Error from persistence.
+    Persist(E),
+    /// Occurs when the loaded changeset cannot contruct [`Wallet`].
+    Descriptor(DescriptorError),
+}
+
+impl<E: fmt::Display> fmt::Display for CreateWithPersistError<E> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Persist(err) => fmt::Display::fmt(err, f),
+            Self::Descriptor(err) => fmt::Display::fmt(&err, f),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl<E: fmt::Debug + fmt::Display> std::error::Error for CreateWithPersistError<E> {}

crates/wallet/src/wallet/signer.rs

@@ -19,13 +19,12 @@
 //! # use core::str::FromStr;
 //! # use bitcoin::secp256k1::{Secp256k1, All};
 //! # use bitcoin::*;
-//! # use bitcoin::psbt;
-//! # use bdk::signer::*;
-//! # use bdk::*;
+//! # use bdk_wallet::signer::*;
+//! # use bdk_wallet::*;
 //! # #[derive(Debug)]
 //! # struct CustomHSM;
 //! # impl CustomHSM {
-//! #     fn hsm_sign_input(&self, _psbt: &mut psbt::PartiallySignedTransaction, _input: usize) -> Result<(), SignerError> {
+//! #     fn hsm_sign_input(&self, _psbt: &mut Psbt, _input: usize) -> Result<(), SignerError> {
 //! #         Ok(())
 //! #     }
 //! #     fn connect() -> Self {
@@ -55,7 +54,7 @@
 //! impl InputSigner for CustomSigner {
 //!     fn sign_input(
 //!         &self,
-//!         psbt: &mut psbt::PartiallySignedTransaction,
+//!         psbt: &mut Psbt,
 //!         input_index: usize,
 //!         _sign_options: &SignOptions,
 //!         _secp: &Secp256k1<All>,
@@ -68,8 +67,11 @@
 //!
 //! let custom_signer = CustomSigner::connect();
 //!
-//! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/*)";
-//! let mut wallet = Wallet::new_no_persist(descriptor, None, Network::Testnet)?;
+//! let descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/0/*)";
+//! let change_descriptor = "wpkh(tpubD6NzVbkrYhZ4Xferm7Pz4VnjdcDPFyjVu5K4iZXQ4pVN8Cks4pHVowTBXBKRhX64pkRyJZJN5xAKj4UDNnLPb5p2sSKXhewoYx5GbTdUFWq/1/*)";
+//! let mut wallet = Wallet::create(descriptor, change_descriptor)
+//!     .network(Network::Testnet)
+//!     .create_wallet_no_persist()?;
 //! wallet.add_signer(
 //!     KeychainKind::External,
 //!     SignerOrdering(200),
@@ -87,19 +89,19 @@ use core::cmp::Ordering;
 use core::fmt;
 use core::ops::{Bound::Included, Deref};
 
-use bitcoin::bip32::{ChildNumber, DerivationPath, ExtendedPrivKey, Fingerprint};
+use bitcoin::bip32::{ChildNumber, DerivationPath, Fingerprint, Xpriv};
 use bitcoin::hashes::hash160;
 use bitcoin::secp256k1::Message;
 use bitcoin::sighash::{EcdsaSighashType, TapSighash, TapSighashType};
 use bitcoin::{ecdsa, psbt, sighash, taproot};
 use bitcoin::{key::TapTweak, key::XOnlyPublicKey, secp256k1};
-use bitcoin::{PrivateKey, PublicKey};
+use bitcoin::{PrivateKey, Psbt, PublicKey};
 
 use miniscript::descriptor::{
     Descriptor, DescriptorMultiXKey, DescriptorPublicKey, DescriptorSecretKey, DescriptorXKey,
     InnerXKey, KeyMap, SinglePriv, SinglePubKey,
 };
-use miniscript::{Legacy, Segwitv0, SigType, Tap, ToPublicKey};
+use miniscript::{SigType, ToPublicKey};
 
 use super::utils::SecpCtx;
 use crate::descriptor::{DescriptorMeta, XKeyUtils};
@@ -159,8 +161,10 @@ pub enum SignerError {
     NonStandardSighash,
     /// Invalid SIGHASH for the signing context in use
     InvalidSighash,
-    /// Error while computing the hash to sign
-    SighashError(sighash::Error),
+    /// Error while computing the hash to sign a Taproot input.
+    SighashTaproot(sighash::TaprootError),
+    /// PSBT sign error.
+    Psbt(psbt::SignError),
     /// Miniscript PSBT error
     MiniscriptPsbt(MiniscriptPsbtError),
     /// To be used only by external libraries implementing [`InputSigner`] or
@@ -169,12 +173,6 @@ pub enum SignerError {
     External(String),
 }
 
-impl From<sighash::Error> for SignerError {
-    fn from(e: sighash::Error) -> Self {
-        SignerError::SighashError(e)
-    }
-}
-
 impl fmt::Display for SignerError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -189,7 +187,8 @@ impl fmt::Display for SignerError {
             Self::MissingHdKeypath => write!(f, "Missing fingerprint and derivation path"),
             Self::NonStandardSighash => write!(f, "The psbt contains a non standard sighash"),
             Self::InvalidSighash => write!(f, "Invalid SIGHASH for the signing context in use"),
-            Self::SighashError(err) => write!(f, "Error while computing the hash to sign: {}", err),
+            Self::SighashTaproot(err) => write!(f, "Error while computing the hash to sign a Taproot input: {}", err),
+            Self::Psbt(err) => write!(f, "Error computing the sighash: {}", err),
             Self::MiniscriptPsbt(err) => write!(f, "Miniscript PSBT error: {}", err),
             Self::External(err) => write!(f, "{}", err),
         }
@@ -264,7 +263,7 @@ pub trait InputSigner: SignerCommon {
     /// Sign a single psbt input
     fn sign_input(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         input_index: usize,
         sign_options: &SignOptions,
         secp: &SecpCtx,
@@ -279,7 +278,7 @@ pub trait TransactionSigner: SignerCommon {
     /// Sign all the inputs of the psbt
     fn sign_transaction(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         sign_options: &SignOptions,
         secp: &SecpCtx,
     ) -> Result<(), SignerError>;
@@ -288,7 +287,7 @@ pub trait TransactionSigner: SignerCommon {
 impl<T: InputSigner> TransactionSigner for T {
     fn sign_transaction(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         sign_options: &SignOptions,
         secp: &SecpCtx,
     ) -> Result<(), SignerError> {
@@ -300,7 +299,7 @@ impl<T: InputSigner> TransactionSigner for T {
     }
 }
 
-impl SignerCommon for SignerWrapper<DescriptorXKey<ExtendedPrivKey>> {
+impl SignerCommon for SignerWrapper<DescriptorXKey<Xpriv>> {
     fn id(&self, secp: &SecpCtx) -> SignerId {
         SignerId::from(self.root_fingerprint(secp))
     }
@@ -310,10 +309,10 @@ impl SignerCommon for SignerWrapper<DescriptorXKey<ExtendedPrivKey>> {
     }
 }
 
-impl InputSigner for SignerWrapper<DescriptorXKey<ExtendedPrivKey>> {
+impl InputSigner for SignerWrapper<DescriptorXKey<Xpriv>> {
     fn sign_input(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         input_index: usize,
         sign_options: &SignOptions,
         secp: &SecpCtx,
@@ -396,7 +395,7 @@ fn multikey_to_xkeys<K: InnerXKey + Clone>(
         .collect()
 }
 
-impl SignerCommon for SignerWrapper<DescriptorMultiXKey<ExtendedPrivKey>> {
+impl SignerCommon for SignerWrapper<DescriptorMultiXKey<Xpriv>> {
     fn id(&self, secp: &SecpCtx) -> SignerId {
         SignerId::from(self.root_fingerprint(secp))
     }
@@ -406,10 +405,10 @@ impl SignerCommon for SignerWrapper<DescriptorMultiXKey<ExtendedPrivKey>> {
     }
 }
 
-impl InputSigner for SignerWrapper<DescriptorMultiXKey<ExtendedPrivKey>> {
+impl InputSigner for SignerWrapper<DescriptorMultiXKey<Xpriv>> {
     fn sign_input(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         input_index: usize,
         sign_options: &SignOptions,
         secp: &SecpCtx,
@@ -438,7 +437,7 @@ impl SignerCommon for SignerWrapper<PrivateKey> {
 impl InputSigner for SignerWrapper<PrivateKey> {
     fn sign_input(
         &self,
-        psbt: &mut psbt::PartiallySignedTransaction,
+        psbt: &mut Psbt,
         input_index: usize,
         sign_options: &SignOptions,
         secp: &SecpCtx,
@@ -454,93 +453,88 @@ impl InputSigner for SignerWrapper<PrivateKey> {
         }
 
         let pubkey = PublicKey::from_private_key(secp, self);
-        let x_only_pubkey = XOnlyPublicKey::from(pubkey.inner);
 
-        if let SignerContext::Tap { is_internal_key } = self.ctx {
-            if let Some(psbt_internal_key) = psbt.inputs[input_index].tap_internal_key {
-                if is_internal_key
-                    && psbt.inputs[input_index].tap_key_sig.is_none()
-                    && sign_options.sign_with_tap_internal_key
-                    && x_only_pubkey == psbt_internal_key
+        match self.ctx {
+            SignerContext::Tap { is_internal_key } => {
+                let x_only_pubkey = XOnlyPublicKey::from(pubkey.inner);
+
+                if let Some(psbt_internal_key) = psbt.inputs[input_index].tap_internal_key {
+                    if is_internal_key
+                        && psbt.inputs[input_index].tap_key_sig.is_none()
+                        && sign_options.sign_with_tap_internal_key
+                        && x_only_pubkey == psbt_internal_key
+                    {
+                        let (sighash, sighash_type) = compute_tap_sighash(psbt, input_index, None)?;
+                        sign_psbt_schnorr(
+                            &self.inner,
+                            x_only_pubkey,
+                            None,
+                            &mut psbt.inputs[input_index],
+                            sighash,
+                            sighash_type,
+                            secp,
+                        );
+                    }
+                }
+
+                if let Some((leaf_hashes, _)) =
+                    psbt.inputs[input_index].tap_key_origins.get(&x_only_pubkey)
                 {
-                    let (hash, hash_ty) = Tap::sighash(psbt, input_index, None)?;
-                    sign_psbt_schnorr(
-                        &self.inner,
-                        x_only_pubkey,
-                        None,
-                        &mut psbt.inputs[input_index],
-                        hash,
-                        hash_ty,
-                        secp,
-                    );
+                    let leaf_hashes = leaf_hashes
+                        .iter()
+                        .filter(|lh| {
+                            // Removing the leaves we shouldn't sign for
+                            let should_sign = match &sign_options.tap_leaves_options {
+                                TapLeavesOptions::All => true,
+                                TapLeavesOptions::Include(v) => v.contains(lh),
+                                TapLeavesOptions::Exclude(v) => !v.contains(lh),
+                                TapLeavesOptions::None => false,
+                            };
+                            // Filtering out the leaves without our key
+                            should_sign
+                                && !psbt.inputs[input_index]
+                                    .tap_script_sigs
+                                    .contains_key(&(x_only_pubkey, **lh))
+                        })
+                        .cloned()
+                        .collect::<Vec<_>>();
+                    for lh in leaf_hashes {
+                        let (sighash, sighash_type) =
+                            compute_tap_sighash(psbt, input_index, Some(lh))?;
+                        sign_psbt_schnorr(
+                            &self.inner,
+                            x_only_pubkey,
+                            Some(lh),
+                            &mut psbt.inputs[input_index],
+                            sighash,
+                            sighash_type,
+                            secp,
+                        );
+                    }
                 }
             }
-
-            if let Some((leaf_hashes, _)) =
-                psbt.inputs[input_index].tap_key_origins.get(&x_only_pubkey)
-            {
-                let leaf_hashes = leaf_hashes
-                    .iter()
-                    .filter(|lh| {
-                        // Removing the leaves we shouldn't sign for
-                        let should_sign = match &sign_options.tap_leaves_options {
-                            TapLeavesOptions::All => true,
-                            TapLeavesOptions::Include(v) => v.contains(lh),
-                            TapLeavesOptions::Exclude(v) => !v.contains(lh),
-                            TapLeavesOptions::None => false,
-                        };
-                        // Filtering out the leaves without our key
-                        should_sign
-                            && !psbt.inputs[input_index]
-                                .tap_script_sigs
-                                .contains_key(&(x_only_pubkey, **lh))
-                    })
-                    .cloned()
-                    .collect::<Vec<_>>();
-                for lh in leaf_hashes {
-                    let (hash, hash_ty) = Tap::sighash(psbt, input_index, Some(lh))?;
-                    sign_psbt_schnorr(
-                        &self.inner,
-                        x_only_pubkey,
-                        Some(lh),
-                        &mut psbt.inputs[input_index],
-                        hash,
-                        hash_ty,
-                        secp,
-                    );
+            SignerContext::Segwitv0 | SignerContext::Legacy => {
+                if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
+                    return Ok(());
                 }
-            }
 
-            return Ok(());
+                let mut sighasher = sighash::SighashCache::new(psbt.unsigned_tx.clone());
+                let (msg, sighash_type) = psbt
+                    .sighash_ecdsa(input_index, &mut sighasher)
+                    .map_err(SignerError::Psbt)?;
+
+                sign_psbt_ecdsa(
+                    &self.inner,
+                    pubkey,
+                    &mut psbt.inputs[input_index],
+                    &msg,
+                    sighash_type,
+                    secp,
+                    sign_options.allow_grinding,
+                );
+            }
         }
 
-        if psbt.inputs[input_index].partial_sigs.contains_key(&pubkey) {
-            return Ok(());
-        }
-
-        let (hash, hash_ty) = match self.ctx {
-            SignerContext::Segwitv0 => {
-                let (h, t) = Segwitv0::sighash(psbt, input_index, ())?;
-                let h = h.to_raw_hash();
-                (h, t)
-            }
-            SignerContext::Legacy => {
-                let (h, t) = Legacy::sighash(psbt, input_index, ())?;
-                let h = h.to_raw_hash();
-                (h, t)
-            }
-            _ => return Ok(()), // handled above
-        };
-        sign_psbt_ecdsa(
-            &self.inner,
-            pubkey,
-            &mut psbt.inputs[input_index],
-            hash,
-            hash_ty,
-            secp,
-            sign_options.allow_grinding,
-        );
-
         Ok(())
     }
 }
@@ -549,21 +543,23 @@ fn sign_psbt_ecdsa(
     secret_key: &secp256k1::SecretKey,
     pubkey: PublicKey,
     psbt_input: &mut psbt::Input,
-    hash: impl bitcoin::hashes::Hash + bitcoin::secp256k1::ThirtyTwoByteHash,
-    hash_ty: EcdsaSighashType,
+    msg: &Message,
+    sighash_type: EcdsaSighashType,
     secp: &SecpCtx,
     allow_grinding: bool,
 ) {
-    let msg = &Message::from(hash);
-    let sig = if allow_grinding {
+    let signature = if allow_grinding {
         secp.sign_ecdsa_low_r(msg, secret_key)
     } else {
         secp.sign_ecdsa(msg, secret_key)
     };
-    secp.verify_ecdsa(msg, &sig, &pubkey.inner)
+    secp.verify_ecdsa(msg, &signature, &pubkey.inner)
         .expect("invalid or corrupted ecdsa signature");
 
-    let final_signature = ecdsa::Signature { sig, hash_ty };
+    let final_signature = ecdsa::Signature {
+        signature,
+        sighash_type,
+    };
     psbt_input.partial_sigs.insert(pubkey, final_signature);
 }
 
@@ -573,11 +569,11 @@ fn sign_psbt_schnorr(
     pubkey: XOnlyPublicKey,
     leaf_hash: Option<taproot::TapLeafHash>,
     psbt_input: &mut psbt::Input,
-    hash: TapSighash,
-    hash_ty: TapSighashType,
+    sighash: TapSighash,
+    sighash_type: TapSighashType,
     secp: &SecpCtx,
 ) {
-    let keypair = secp256k1::KeyPair::from_seckey_slice(secp, secret_key.as_ref()).unwrap();
+    let keypair = secp256k1::Keypair::from_seckey_slice(secp, secret_key.as_ref()).unwrap();
     let keypair = match leaf_hash {
         None => keypair
             .tap_tweak(secp, psbt_input.tap_merkle_root)
@@ -585,12 +581,15 @@ fn sign_psbt_schnorr(
         Some(_) => keypair, // no tweak for script spend
     };
 
-    let msg = &Message::from(hash);
-    let sig = secp.sign_schnorr(msg, &keypair);
-    secp.verify_schnorr(&sig, msg, &XOnlyPublicKey::from_keypair(&keypair).0)
+    let msg = &Message::from(sighash);
+    let signature = secp.sign_schnorr_no_aux_rand(msg, &keypair);
+    secp.verify_schnorr(&signature, msg, &XOnlyPublicKey::from_keypair(&keypair).0)
         .expect("invalid or corrupted schnorr signature");
 
-    let final_signature = taproot::Signature { sig, hash_ty };
+    let final_signature = taproot::Signature {
+        signature,
+        sighash_type,
+    };
 
     if let Some(lh) = leaf_hash {
         psbt_input
@@ -777,11 +776,6 @@ pub struct SignOptions {
     /// Defaults to `false` which will only allow signing using `SIGHASH_ALL`.
     pub allow_all_sighashes: bool,
 
-    /// Whether to remove partial signatures from the PSBT inputs while finalizing PSBT.
-    ///
-    /// Defaults to `true` which will remove partial signatures during finalization.
-    pub remove_partial_sigs: bool,
-
     /// Whether to try finalizing the PSBT after the inputs are signed.
     ///
     /// Defaults to `true` which will try finalizing PSBT after inputs are signed.
@@ -826,7 +820,6 @@ impl Default for SignOptions {
             trust_witness_utxo: false,
             assume_height: None,
             allow_all_sighashes: false,
-            remove_partial_sigs: true,
             try_finalize: true,
             tap_leaves_options: TapLeavesOptions::default(),
             sign_with_tap_internal_key: true,
@@ -835,199 +828,53 @@ impl Default for SignOptions {
     }
 }
 
-pub(crate) trait ComputeSighash {
-    type Extra;
-    type Sighash;
-    type SighashType;
-
-    fn sighash(
-        psbt: &psbt::PartiallySignedTransaction,
-        input_index: usize,
-        extra: Self::Extra,
-    ) -> Result<(Self::Sighash, Self::SighashType), SignerError>;
-}
-
-impl ComputeSighash for Legacy {
-    type Extra = ();
-    type Sighash = sighash::LegacySighash;
-    type SighashType = EcdsaSighashType;
-
-    fn sighash(
-        psbt: &psbt::PartiallySignedTransaction,
-        input_index: usize,
-        _extra: (),
-    ) -> Result<(Self::Sighash, Self::SighashType), SignerError> {
-        if input_index >= psbt.inputs.len() || input_index >= psbt.unsigned_tx.input.len() {
-            return Err(SignerError::InputIndexOutOfRange);
-        }
-
-        let psbt_input = &psbt.inputs[input_index];
-        let tx_input = &psbt.unsigned_tx.input[input_index];
-
-        let sighash = psbt_input
-            .sighash_type
-            .unwrap_or_else(|| EcdsaSighashType::All.into())
-            .ecdsa_hash_ty()
-            .map_err(|_| SignerError::InvalidSighash)?;
-        let script = match psbt_input.redeem_script {
-            Some(ref redeem_script) => redeem_script.clone(),
-            None => {
-                let non_witness_utxo = psbt_input
-                    .non_witness_utxo
-                    .as_ref()
-                    .ok_or(SignerError::MissingNonWitnessUtxo)?;
-                let prev_out = non_witness_utxo
-                    .output
-                    .get(tx_input.previous_output.vout as usize)
-                    .ok_or(SignerError::InvalidNonWitnessUtxo)?;
-
-                prev_out.script_pubkey.clone()
-            }
-        };
-
-        Ok((
-            sighash::SighashCache::new(&psbt.unsigned_tx).legacy_signature_hash(
-                input_index,
-                &script,
-                sighash.to_u32(),
-            )?,
-            sighash,
-        ))
+/// Computes the taproot sighash.
+fn compute_tap_sighash(
+    psbt: &Psbt,
+    input_index: usize,
+    extra: Option<taproot::TapLeafHash>,
+) -> Result<(sighash::TapSighash, TapSighashType), SignerError> {
+    if input_index >= psbt.inputs.len() || input_index >= psbt.unsigned_tx.input.len() {
+        return Err(SignerError::InputIndexOutOfRange);
     }
-}
 
-impl ComputeSighash for Segwitv0 {
-    type Extra = ();
-    type Sighash = sighash::SegwitV0Sighash;
-    type SighashType = EcdsaSighashType;
+    let psbt_input = &psbt.inputs[input_index];
 
-    fn sighash(
-        psbt: &psbt::PartiallySignedTransaction,
-        input_index: usize,
-        _extra: (),
-    ) -> Result<(Self::Sighash, Self::SighashType), SignerError> {
-        if input_index >= psbt.inputs.len() || input_index >= psbt.unsigned_tx.input.len() {
-            return Err(SignerError::InputIndexOutOfRange);
-        }
+    let sighash_type = psbt_input
+        .sighash_type
+        .unwrap_or_else(|| TapSighashType::Default.into())
+        .taproot_hash_ty()
+        .map_err(|_| SignerError::InvalidSighash)?;
+    let witness_utxos = (0..psbt.inputs.len())
+        .map(|i| psbt.get_utxo_for(i))
+        .collect::<Vec<_>>();
+    let mut all_witness_utxos = vec![];
 
-        let psbt_input = &psbt.inputs[input_index];
-        let tx_input = &psbt.unsigned_tx.input[input_index];
+    let mut cache = sighash::SighashCache::new(&psbt.unsigned_tx);
+    let is_anyone_can_pay = psbt::PsbtSighashType::from(sighash_type).to_u32() & 0x80 != 0;
+    let prevouts = if is_anyone_can_pay {
+        sighash::Prevouts::One(
+            input_index,
+            witness_utxos[input_index]
+                .as_ref()
+                .ok_or(SignerError::MissingWitnessUtxo)?,
+        )
+    } else if witness_utxos.iter().all(Option::is_some) {
+        all_witness_utxos.extend(witness_utxos.iter().filter_map(|x| x.as_ref()));
+        sighash::Prevouts::All(&all_witness_utxos)
+    } else {
+        return Err(SignerError::MissingWitnessUtxo);
+    };
 
-        let sighash = psbt_input
-            .sighash_type
-            .unwrap_or_else(|| EcdsaSighashType::All.into())
-            .ecdsa_hash_ty()
-            .map_err(|_| SignerError::InvalidSighash)?;
+    // Assume no OP_CODESEPARATOR
+    let extra = extra.map(|leaf_hash| (leaf_hash, 0xFFFFFFFF));
 
-        // Always try first with the non-witness utxo
-        let utxo = if let Some(prev_tx) = &psbt_input.non_witness_utxo {
-            // Check the provided prev-tx
-            if prev_tx.txid() != tx_input.previous_output.txid {
-                return Err(SignerError::InvalidNonWitnessUtxo);
-            }
-
-            // The output should be present, if it's missing the `non_witness_utxo` is invalid
-            prev_tx
-                .output
-                .get(tx_input.previous_output.vout as usize)
-                .ok_or(SignerError::InvalidNonWitnessUtxo)?
-        } else if let Some(witness_utxo) = &psbt_input.witness_utxo {
-            // Fallback to the witness_utxo. If we aren't allowed to use it, signing should fail
-            // before we get to this point
-            witness_utxo
-        } else {
-            // Nothing has been provided
-            return Err(SignerError::MissingNonWitnessUtxo);
-        };
-        let value = utxo.value;
-
-        let script = match psbt_input.witness_script {
-            Some(ref witness_script) => witness_script.clone(),
-            None => {
-                if utxo.script_pubkey.is_v0_p2wpkh() {
-                    utxo.script_pubkey
-                        .p2wpkh_script_code()
-                        .expect("We check above that the spk is a p2wpkh")
-                } else if psbt_input
-                    .redeem_script
-                    .as_ref()
-                    .map(|s| s.is_v0_p2wpkh())
-                    .unwrap_or(false)
-                {
-                    psbt_input
-                        .redeem_script
-                        .as_ref()
-                        .unwrap()
-                        .p2wpkh_script_code()
-                        .expect("We check above that the spk is a p2wpkh")
-                } else {
-                    return Err(SignerError::MissingWitnessScript);
-                }
-            }
-        };
-
-        Ok((
-            sighash::SighashCache::new(&psbt.unsigned_tx).segwit_signature_hash(
-                input_index,
-                &script,
-                value,
-                sighash,
-            )?,
-            sighash,
-        ))
-    }
-}
-
-impl ComputeSighash for Tap {
-    type Extra = Option<taproot::TapLeafHash>;
-    type Sighash = TapSighash;
-    type SighashType = TapSighashType;
-
-    fn sighash(
-        psbt: &psbt::PartiallySignedTransaction,
-        input_index: usize,
-        extra: Self::Extra,
-    ) -> Result<(Self::Sighash, TapSighashType), SignerError> {
-        if input_index >= psbt.inputs.len() || input_index >= psbt.unsigned_tx.input.len() {
-            return Err(SignerError::InputIndexOutOfRange);
-        }
-
-        let psbt_input = &psbt.inputs[input_index];
-
-        let sighash_type = psbt_input
-            .sighash_type
-            .unwrap_or_else(|| TapSighashType::Default.into())
-            .taproot_hash_ty()
-            .map_err(|_| SignerError::InvalidSighash)?;
-        let witness_utxos = (0..psbt.inputs.len())
-            .map(|i| psbt.get_utxo_for(i))
-            .collect::<Vec<_>>();
-        let mut all_witness_utxos = vec![];
-
-        let mut cache = sighash::SighashCache::new(&psbt.unsigned_tx);
-        let is_anyone_can_pay = psbt::PsbtSighashType::from(sighash_type).to_u32() & 0x80 != 0;
-        let prevouts = if is_anyone_can_pay {
-            sighash::Prevouts::One(
-                input_index,
-                witness_utxos[input_index]
-                    .as_ref()
-                    .ok_or(SignerError::MissingWitnessUtxo)?,
-            )
-        } else if witness_utxos.iter().all(Option::is_some) {
-            all_witness_utxos.extend(witness_utxos.iter().filter_map(|x| x.as_ref()));
-            sighash::Prevouts::All(&all_witness_utxos)
-        } else {
-            return Err(SignerError::MissingWitnessUtxo);
-        };
-
-        // Assume no OP_CODESEPARATOR
-        let extra = extra.map(|leaf_hash| (leaf_hash, 0xFFFFFFFF));
-
-        Ok((
-            cache.taproot_signature_hash(input_index, &prevouts, None, extra, sighash_type)?,
-            sighash_type,
-        ))
-    }
+    Ok((
+        cache
+            .taproot_signature_hash(input_index, &prevouts, None, extra, sighash_type)
+            .map_err(SignerError::SighashTaproot)?,
+        sighash_type,
+    ))
 }
 
 impl PartialOrd for SignersContainerKey {
@@ -1155,7 +1002,7 @@ mod signers_container_tests {
     impl TransactionSigner for DummySigner {
         fn sign_transaction(
             &self,
-            _psbt: &mut psbt::PartiallySignedTransaction,
+            _psbt: &mut Psbt,
             _sign_options: &SignOptions,
             _secp: &SecpCtx,
         ) -> Result<(), SignerError> {
@@ -1173,8 +1020,8 @@ mod signers_container_tests {
     ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
         let secp: Secp256k1<All> = Secp256k1::new();
         let path = bip32::DerivationPath::from_str(PATH).unwrap();
-        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
-        let tpub = bip32::ExtendedPubKey::from_priv(&secp, &tprv);
+        let tprv = bip32::Xpriv::from_str(tprv).unwrap();
+        let tpub = bip32::Xpub::from_priv(&secp, &tprv);
         let fingerprint = tprv.fingerprint(&secp);
         let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
         let pubkey = (tpub, path).into_descriptor_key().unwrap();

crates/wallet/src/wallet/tx_builder.rs

@@ -16,11 +16,9 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::*;
-//! # use bdk::wallet::ChangeSet;
-//! # use bdk::wallet::error::CreateTxError;
-//! # use bdk::wallet::tx_builder::CreateTx;
-//! # use bdk_chain::PersistBackend;
+//! # use bdk_wallet::*;
+//! # use bdk_wallet::ChangeSet;
+//! # use bdk_wallet::error::CreateTxError;
 //! # use anyhow::Error;
 //! # let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap().assume_checked();
 //! # let mut wallet = doctest_wallet!();
@@ -29,9 +27,9 @@
 //!
 //! tx_builder
 //!     // Create a transaction with one output to `to_address` of 50_000 satoshi
-//!     .add_recipient(to_address.script_pubkey(), 50_000)
+//!     .add_recipient(to_address.script_pubkey(), Amount::from_sat(50_000))
 //!     // With a custom fee rate of 5.0 satoshi/vbyte
-//!     .fee_rate(bdk::FeeRate::from_sat_per_vb(5.0))
+//!     .fee_rate(FeeRate::from_sat_per_vb(5).expect("valid feerate"))
 //!     // Only spend non-change outputs
 //!     .do_not_spend_change()
 //!     // Turn on RBF signaling
@@ -40,36 +38,25 @@
 //! # Ok::<(), anyhow::Error>(())
 //! ```
 
-use crate::collections::BTreeMap;
-use crate::collections::HashSet;
 use alloc::{boxed::Box, rc::Rc, string::String, vec::Vec};
-use bdk_chain::PersistBackend;
 use core::cell::RefCell;
 use core::fmt;
-use core::marker::PhantomData;
 
-use bitcoin::psbt::{self, PartiallySignedTransaction as Psbt};
-use bitcoin::{absolute, script::PushBytes, OutPoint, ScriptBuf, Sequence, Transaction, Txid};
+use alloc::sync::Arc;
 
-use super::coin_selection::{CoinSelectionAlgorithm, DefaultCoinSelectionAlgorithm};
-use super::ChangeSet;
-use crate::types::{FeeRate, KeychainKind, LocalOutput, WeightedUtxo};
-use crate::wallet::CreateTxError;
-use crate::{Utxo, Wallet};
+use bitcoin::psbt::{self, Psbt};
+use bitcoin::script::PushBytes;
+use bitcoin::{
+    absolute, Amount, FeeRate, OutPoint, ScriptBuf, Sequence, Transaction, TxIn, TxOut, Txid,
+    Weight,
+};
+use rand_core::RngCore;
 
-/// Context in which the [`TxBuilder`] is valid
-pub trait TxBuilderContext: core::fmt::Debug + Default + Clone {}
-
-/// Marker type to indicate the [`TxBuilder`] is being used to create a new transaction (as opposed
-/// to bumping the fee of an existing one).
-#[derive(Debug, Default, Clone)]
-pub struct CreateTx;
-impl TxBuilderContext for CreateTx {}
-
-/// Marker type to indicate the [`TxBuilder`] is being used to bump the fee of an existing transaction.
-#[derive(Debug, Default, Clone)]
-pub struct BumpFee;
-impl TxBuilderContext for BumpFee {}
+use super::coin_selection::CoinSelectionAlgorithm;
+use super::utils::shuffle_slice;
+use super::{CreateTxError, Wallet};
+use crate::collections::{BTreeMap, HashSet};
+use crate::{KeychainKind, LocalOutput, Utxo, WeightedUtxo};
 
 /// A transaction builder
 ///
@@ -81,13 +68,12 @@ impl TxBuilderContext for BumpFee {}
 /// as in the following example:
 ///
 /// ```
-/// # use bdk::*;
-/// # use bdk::wallet::tx_builder::*;
+/// # use bdk_wallet::*;
+/// # use bdk_wallet::tx_builder::*;
 /// # use bitcoin::*;
 /// # use core::str::FromStr;
-/// # use bdk::wallet::ChangeSet;
-/// # use bdk::wallet::error::CreateTxError;
-/// # use bdk_chain::PersistBackend;
+/// # use bdk_wallet::ChangeSet;
+/// # use bdk_wallet::error::CreateTxError;
 /// # use anyhow::Error;
 /// # let mut wallet = doctest_wallet!();
 /// # let addr1 = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap().assume_checked();
@@ -97,8 +83,8 @@ impl TxBuilderContext for BumpFee {}
 ///     let mut builder = wallet.build_tx();
 ///     builder
 ///         .ordering(TxOrdering::Untouched)
-///         .add_recipient(addr1.script_pubkey(), 50_000)
-///         .add_recipient(addr2.script_pubkey(), 50_000);
+///         .add_recipient(addr1.script_pubkey(), Amount::from_sat(50_000))
+///         .add_recipient(addr2.script_pubkey(), Amount::from_sat(50_000));
 ///     builder.finish()?
 /// };
 ///
@@ -107,7 +93,7 @@ impl TxBuilderContext for BumpFee {}
 ///     let mut builder = wallet.build_tx();
 ///     builder.ordering(TxOrdering::Untouched);
 ///     for addr in &[addr1, addr2] {
-///         builder.add_recipient(addr.script_pubkey(), 50_000);
+///         builder.add_recipient(addr.script_pubkey(), Amount::from_sat(50_000));
 ///     }
 ///     builder.finish()?
 /// };
@@ -126,11 +112,10 @@ impl TxBuilderContext for BumpFee {}
 /// [`finish`]: Self::finish
 /// [`coin_selection`]: Self::coin_selection
 #[derive(Debug)]
-pub struct TxBuilder<'a, D, Cs, Ctx> {
-    pub(crate) wallet: Rc<RefCell<&'a mut Wallet<D>>>,
+pub struct TxBuilder<'a, Cs> {
+    pub(crate) wallet: Rc<RefCell<&'a mut Wallet>>,
     pub(crate) params: TxParams,
     pub(crate) coin_selection: Cs,
-    pub(crate) phantom: PhantomData<Ctx>,
 }
 
 /// The parameters for transaction creation sans coin selection algorithm.
@@ -163,7 +148,7 @@ pub(crate) struct TxParams {
 #[derive(Clone, Copy, Debug)]
 pub(crate) struct PreviousFee {
     pub absolute: u64,
-    pub rate: f32,
+    pub rate: FeeRate,
 }
 
 #[derive(Debug, Clone, Copy)]
@@ -174,31 +159,28 @@ pub(crate) enum FeePolicy {
 
 impl Default for FeePolicy {
     fn default() -> Self {
-        FeePolicy::FeeRate(FeeRate::default_min_relay_fee())
+        FeePolicy::FeeRate(FeeRate::BROADCAST_MIN)
     }
 }
 
-impl<'a, D, Cs: Clone, Ctx> Clone for TxBuilder<'a, D, Cs, Ctx> {
+impl<'a, Cs: Clone> Clone for TxBuilder<'a, Cs> {
     fn clone(&self) -> Self {
         TxBuilder {
             wallet: self.wallet.clone(),
             params: self.params.clone(),
             coin_selection: self.coin_selection.clone(),
-            phantom: PhantomData,
         }
     }
 }
 
-// methods supported by both contexts, for any CoinSelectionAlgorithm
-impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D, Cs, Ctx> {
-    /// Set a custom fee rate
-    /// The fee_rate method sets the mining fee paid by the transaction as a rate on its size.
-    /// This means that the total fee paid is equal to this rate * size of the transaction in virtual Bytes (vB) or Weight Unit (wu).
-    /// This rate is internally expressed in satoshis-per-virtual-bytes (sats/vB) using FeeRate::from_sat_per_vb, but can also be set by:
-    /// * sats/kvB (1000 sats/kvB == 1 sats/vB) using FeeRate::from_sat_per_kvb
-    /// * btc/kvB (0.00001000 btc/kvB == 1 sats/vB) using FeeRate::from_btc_per_kvb
-    /// * sats/kwu (250 sats/kwu == 1 sats/vB) using FeeRate::from_sat_per_kwu
-    /// Default is 1 sat/vB (see min_relay_fee)
+// Methods supported for any CoinSelectionAlgorithm.
+impl<'a, Cs> TxBuilder<'a, Cs> {
+    /// Set a custom fee rate.
+    ///
+    /// This method sets the mining fee paid by the transaction as a rate on its size.
+    /// This means that the total fee paid is equal to `fee_rate` times the size
+    /// of the transaction. Default is 1 sat/vB in accordance with Bitcoin Core's default
+    /// relay policy.
     ///
     /// Note that this is really a minimum feerate -- it's possible to
     /// overshoot it slightly since adding a change output to drain the remaining
@@ -209,16 +191,16 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     }
 
     /// Set an absolute fee
-    /// The fee_absolute method refers to the absolute transaction fee in satoshis (sats).
-    /// If anyone sets both the fee_absolute method and the fee_rate method,
-    /// the FeePolicy enum will be set by whichever method was called last,
-    /// as the FeeRate and FeeAmount are mutually exclusive.
+    /// The fee_absolute method refers to the absolute transaction fee in [`Amount`].
+    /// If anyone sets both the `fee_absolute` method and the `fee_rate` method,
+    /// the `FeePolicy` enum will be set by whichever method was called last,
+    /// as the [`FeeRate`] and `FeeAmount` are mutually exclusive.
     ///
     /// Note that this is really a minimum absolute fee -- it's possible to
     /// overshoot it slightly since adding a change output to drain the remaining
     /// excess might not be viable.
-    pub fn fee_absolute(&mut self, fee_amount: u64) -> &mut Self {
-        self.params.fee_policy = Some(FeePolicy::FeeAmount(fee_amount));
+    pub fn fee_absolute(&mut self, fee_amount: Amount) -> &mut Self {
+        self.params.fee_policy = Some(FeePolicy::FeeAmount(fee_amount.to_sat()));
         self
     }
 
@@ -268,7 +250,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     /// # use std::str::FromStr;
     /// # use std::collections::BTreeMap;
     /// # use bitcoin::*;
-    /// # use bdk::*;
+    /// # use bdk_wallet::*;
     /// # let to_address =
     /// Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
     ///     .unwrap()
@@ -279,7 +261,7 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// let builder = wallet
     ///     .build_tx()
-    ///     .add_recipient(to_address.script_pubkey(), 50_000)
+    ///     .add_recipient(to_address.script_pubkey(), Amount::from_sat(50_000))
     ///     .policy_path(path, KeychainKind::External);
     ///
     /// # Ok::<(), anyhow::Error>(())
@@ -317,9 +299,8 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
                 .collect::<Result<Vec<_>, _>>()?;
 
             for utxo in utxos {
-                let descriptor = wallet.get_descriptor_for_keychain(utxo.keychain);
-                #[allow(deprecated)]
-                let satisfaction_weight = descriptor.max_satisfaction_weight().unwrap();
+                let descriptor = wallet.public_descriptor(utxo.keychain);
+                let satisfaction_weight = descriptor.max_weight_to_satisfy().unwrap();
                 self.params.utxos.push(WeightedUtxo {
                     satisfaction_weight,
                     utxo: Utxo::Local(utxo),
@@ -360,9 +341,9 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     /// causing you to pay a fee that is too high. The party who is broadcasting the transaction can
     /// of course check the real input weight matches the expected weight prior to broadcasting.
     ///
-    /// To guarantee the `satisfaction_weight` is correct, you can require the party providing the
+    /// To guarantee the `max_weight_to_satisfy` is correct, you can require the party providing the
     /// `psbt_input` provide a miniscript descriptor for the input so you can check it against the
-    /// `script_pubkey` and then ask it for the [`max_satisfaction_weight`].
+    /// `script_pubkey` and then ask it for the [`max_weight_to_satisfy`].
     ///
     /// This is an **EXPERIMENTAL** feature, API and other major changes are expected.
     ///
@@ -383,12 +364,12 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
     ///
     /// [`only_witness_utxo`]: Self::only_witness_utxo
     /// [`finish`]: Self::finish
-    /// [`max_satisfaction_weight`]: miniscript::Descriptor::max_satisfaction_weight
+    /// [`max_weight_to_satisfy`]: miniscript::Descriptor::max_weight_to_satisfy
     pub fn add_foreign_utxo(
         &mut self,
         outpoint: OutPoint,
         psbt_input: psbt::Input,
-        satisfaction_weight: usize,
+        satisfaction_weight: Weight,
     ) -> Result<&mut Self, AddForeignUtxoError> {
         self.add_foreign_utxo_with_sequence(
             outpoint,
@@ -403,15 +384,15 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         &mut self,
         outpoint: OutPoint,
         psbt_input: psbt::Input,
-        satisfaction_weight: usize,
+        satisfaction_weight: Weight,
         sequence: Sequence,
     ) -> Result<&mut Self, AddForeignUtxoError> {
         if psbt_input.witness_utxo.is_none() {
             match psbt_input.non_witness_utxo.as_ref() {
                 Some(tx) => {
-                    if tx.txid() != outpoint.txid {
+                    if tx.compute_txid() != outpoint.txid {
                         return Err(AddForeignUtxoError::InvalidTxid {
-                            input_txid: tx.txid(),
+                            input_txid: tx.compute_txid(),
                             foreign_utxo: outpoint,
                         });
                     }
@@ -559,35 +540,17 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
 
     /// Choose the coin selection algorithm
     ///
-    /// Overrides the [`DefaultCoinSelectionAlgorithm`].
+    /// Overrides the [`CoinSelectionAlgorithm`].
     ///
     /// Note that this function consumes the builder and returns it so it is usually best to put this as the first call on the builder.
-    pub fn coin_selection<P: CoinSelectionAlgorithm>(
-        self,
-        coin_selection: P,
-    ) -> TxBuilder<'a, D, P, Ctx> {
+    pub fn coin_selection<P: CoinSelectionAlgorithm>(self, coin_selection: P) -> TxBuilder<'a, P> {
         TxBuilder {
             wallet: self.wallet,
             params: self.params,
             coin_selection,
-            phantom: PhantomData,
         }
     }
 
-    /// Finish building the transaction.
-    ///
-    /// Returns a new [`Psbt`] per [`BIP174`].
-    ///
-    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
-    pub fn finish(self) -> Result<Psbt, CreateTxError<D::WriteError>>
-    where
-        D: PersistBackend<ChangeSet>,
-    {
-        self.wallet
-            .borrow_mut()
-            .create_tx(self.coin_selection, self.params)
-    }
-
     /// Enable signaling RBF
     ///
     /// This will use the default nSequence value of `0xFFFFFFFD`.
@@ -632,6 +595,113 @@ impl<'a, D, Cs: CoinSelectionAlgorithm, Ctx: TxBuilderContext> TxBuilder<'a, D,
         self.params.allow_dust = allow_dust;
         self
     }
+
+    /// Replace the recipients already added with a new list
+    pub fn set_recipients(&mut self, recipients: Vec<(ScriptBuf, Amount)>) -> &mut Self {
+        self.params.recipients = recipients
+            .into_iter()
+            .map(|(script, amount)| (script, amount.to_sat()))
+            .collect();
+        self
+    }
+
+    /// Add a recipient to the internal list
+    pub fn add_recipient(&mut self, script_pubkey: ScriptBuf, amount: Amount) -> &mut Self {
+        self.params
+            .recipients
+            .push((script_pubkey, amount.to_sat()));
+        self
+    }
+
+    /// Add data as an output, using OP_RETURN
+    pub fn add_data<T: AsRef<PushBytes>>(&mut self, data: &T) -> &mut Self {
+        let script = ScriptBuf::new_op_return(data);
+        self.add_recipient(script, Amount::ZERO);
+        self
+    }
+
+    /// Sets the address to *drain* excess coins to.
+    ///
+    /// Usually, when there are excess coins they are sent to a change address generated by the
+    /// wallet. This option replaces the usual change address with an arbitrary `script_pubkey` of
+    /// your choosing. Just as with a change output, if the drain output is not needed (the excess
+    /// coins are too small) it will not be included in the resulting transaction. The only
+    /// difference is that it is valid to use `drain_to` without setting any ordinary recipients
+    /// with [`add_recipient`] (but it is perfectly fine to add recipients as well).
+    ///
+    /// If you choose not to set any recipients, you should provide the utxos that the
+    /// transaction should spend via [`add_utxos`].
+    ///
+    /// # Example
+    ///
+    /// `drain_to` is very useful for draining all the coins in a wallet with [`drain_wallet`] to a
+    /// single address.
+    ///
+    /// ```
+    /// # use std::str::FromStr;
+    /// # use bitcoin::*;
+    /// # use bdk_wallet::*;
+    /// # use bdk_wallet::ChangeSet;
+    /// # use bdk_wallet::error::CreateTxError;
+    /// # use anyhow::Error;
+    /// # let to_address =
+    /// Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
+    ///     .unwrap()
+    ///     .assume_checked();
+    /// # let mut wallet = doctest_wallet!();
+    /// let mut tx_builder = wallet.build_tx();
+    ///
+    /// tx_builder
+    ///     // Spend all outputs in this wallet.
+    ///     .drain_wallet()
+    ///     // Send the excess (which is all the coins minus the fee) to this address.
+    ///     .drain_to(to_address.script_pubkey())
+    ///     .fee_rate(FeeRate::from_sat_per_vb(5).expect("valid feerate"))
+    ///     .enable_rbf();
+    /// let psbt = tx_builder.finish()?;
+    /// # Ok::<(), anyhow::Error>(())
+    /// ```
+    ///
+    /// [`add_recipient`]: Self::add_recipient
+    /// [`add_utxos`]: Self::add_utxos
+    /// [`drain_wallet`]: Self::drain_wallet
+    pub fn drain_to(&mut self, script_pubkey: ScriptBuf) -> &mut Self {
+        self.params.drain_to = Some(script_pubkey);
+        self
+    }
+}
+
+impl<'a, Cs: CoinSelectionAlgorithm> TxBuilder<'a, Cs> {
+    /// Finish building the transaction.
+    ///
+    /// Uses the thread-local random number generator (rng).
+    ///
+    /// Returns a new [`Psbt`] per [`BIP174`].
+    ///
+    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
+    ///
+    /// **WARNING**: To avoid change address reuse you must persist the changes resulting from one
+    /// or more calls to this method before closing the wallet. See [`Wallet::reveal_next_address`].
+    #[cfg(feature = "std")]
+    pub fn finish(self) -> Result<Psbt, CreateTxError> {
+        self.finish_with_aux_rand(&mut bitcoin::key::rand::thread_rng())
+    }
+
+    /// Finish building the transaction.
+    ///
+    /// Uses a provided random number generator (rng).
+    ///
+    /// Returns a new [`Psbt`] per [`BIP174`].
+    ///
+    /// [`BIP174`]: https://github.com/bitcoin/bips/blob/master/bip-0174.mediawiki
+    ///
+    /// **WARNING**: To avoid change address reuse you must persist the changes resulting from one
+    /// or more calls to this method before closing the wallet. See [`Wallet::reveal_next_address`].
+    pub fn finish_with_aux_rand(self, rng: &mut impl RngCore) -> Result<Psbt, CreateTxError> {
+        self.wallet
+            .borrow_mut()
+            .create_tx(self.coin_selection, self.params, rng)
+    }
 }
 
 #[derive(Debug)]
@@ -696,166 +766,60 @@ impl fmt::Display for AddForeignUtxoError {
 #[cfg(feature = "std")]
 impl std::error::Error for AddForeignUtxoError {}
 
-#[derive(Debug)]
-/// Error returned from [`TxBuilder::allow_shrinking`]
-pub enum AllowShrinkingError {
-    /// Script/PubKey was not in the original transaction
-    MissingScriptPubKey(ScriptBuf),
-}
-
-impl fmt::Display for AllowShrinkingError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::MissingScriptPubKey(script_buf) => write!(
-                f,
-                "Script/PubKey was not in the original transaction: {}",
-                script_buf,
-            ),
-        }
-    }
-}
-
-#[cfg(feature = "std")]
-impl std::error::Error for AllowShrinkingError {}
-
-impl<'a, D, Cs: CoinSelectionAlgorithm> TxBuilder<'a, D, Cs, CreateTx> {
-    /// Replace the recipients already added with a new list
-    pub fn set_recipients(&mut self, recipients: Vec<(ScriptBuf, u64)>) -> &mut Self {
-        self.params.recipients = recipients;
-        self
-    }
-
-    /// Add a recipient to the internal list
-    pub fn add_recipient(&mut self, script_pubkey: ScriptBuf, amount: u64) -> &mut Self {
-        self.params.recipients.push((script_pubkey, amount));
-        self
-    }
-
-    /// Add data as an output, using OP_RETURN
-    pub fn add_data<T: AsRef<PushBytes>>(&mut self, data: &T) -> &mut Self {
-        let script = ScriptBuf::new_op_return(data);
-        self.add_recipient(script, 0u64);
-        self
-    }
-
-    /// Sets the address to *drain* excess coins to.
-    ///
-    /// Usually, when there are excess coins they are sent to a change address generated by the
-    /// wallet. This option replaces the usual change address with an arbitrary `script_pubkey` of
-    /// your choosing. Just as with a change output, if the drain output is not needed (the excess
-    /// coins are too small) it will not be included in the resulting transaction. The only
-    /// difference is that it is valid to use `drain_to` without setting any ordinary recipients
-    /// with [`add_recipient`] (but it is perfectly fine to add recipients as well).
-    ///
-    /// If you choose not to set any recipients, you should either provide the utxos that the
-    /// transaction should spend via [`add_utxos`], or set [`drain_wallet`] to spend all of them.
-    ///
-    /// When bumping the fees of a transaction made with this option, you probably want to
-    /// use [`allow_shrinking`] to allow this output to be reduced to pay for the extra fees.
-    ///
-    /// # Example
-    ///
-    /// `drain_to` is very useful for draining all the coins in a wallet with [`drain_wallet`] to a
-    /// single address.
-    ///
-    /// ```
-    /// # use std::str::FromStr;
-    /// # use bitcoin::*;
-    /// # use bdk::*;
-    /// # use bdk::wallet::ChangeSet;
-    /// # use bdk::wallet::error::CreateTxError;
-    /// # use bdk::wallet::tx_builder::CreateTx;
-    /// # use bdk_chain::PersistBackend;
-    /// # use anyhow::Error;
-    /// # let to_address =
-    /// Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
-    ///     .unwrap()
-    ///     .assume_checked();
-    /// # let mut wallet = doctest_wallet!();
-    /// let mut tx_builder = wallet.build_tx();
-    ///
-    /// tx_builder
-    ///     // Spend all outputs in this wallet.
-    ///     .drain_wallet()
-    ///     // Send the excess (which is all the coins minus the fee) to this address.
-    ///     .drain_to(to_address.script_pubkey())
-    ///     .fee_rate(bdk::FeeRate::from_sat_per_vb(5.0))
-    ///     .enable_rbf();
-    /// let psbt = tx_builder.finish()?;
-    /// # Ok::<(), anyhow::Error>(())
-    /// ```
-    ///
-    /// [`allow_shrinking`]: Self::allow_shrinking
-    /// [`add_recipient`]: Self::add_recipient
-    /// [`add_utxos`]: Self::add_utxos
-    /// [`drain_wallet`]: Self::drain_wallet
-    pub fn drain_to(&mut self, script_pubkey: ScriptBuf) -> &mut Self {
-        self.params.drain_to = Some(script_pubkey);
-        self
-    }
-}
-
-// methods supported only by bump_fee
-impl<'a, D> TxBuilder<'a, D, DefaultCoinSelectionAlgorithm, BumpFee> {
-    /// Explicitly tells the wallet that it is allowed to reduce the amount of the output matching this
-    /// `script_pubkey` in order to bump the transaction fee. Without specifying this the wallet
-    /// will attempt to find a change output to shrink instead.
-    ///
-    /// **Note** that the output may shrink to below the dust limit and therefore be removed. If it is
-    /// preserved then it is currently not guaranteed to be in the same position as it was
-    /// originally.
-    ///
-    /// Returns an `Err` if `script_pubkey` can't be found among the recipients of the
-    /// transaction we are bumping.
-    pub fn allow_shrinking(
-        &mut self,
-        script_pubkey: ScriptBuf,
-    ) -> Result<&mut Self, AllowShrinkingError> {
-        match self
-            .params
-            .recipients
-            .iter()
-            .position(|(recipient_script, _)| *recipient_script == script_pubkey)
-        {
-            Some(position) => {
-                self.params.recipients.remove(position);
-                self.params.drain_to = Some(script_pubkey);
-                Ok(self)
-            }
-            None => Err(AllowShrinkingError::MissingScriptPubKey(script_pubkey)),
-        }
-    }
-}
+type TxSort<T> = dyn Fn(&T, &T) -> core::cmp::Ordering;
 
 /// Ordering of the transaction's inputs and outputs
-#[derive(Default, Debug, Ord, PartialOrd, Eq, PartialEq, Hash, Clone, Copy)]
+#[derive(Clone, Default)]
 pub enum TxOrdering {
     /// Randomized (default)
     #[default]
     Shuffle,
     /// Unchanged
     Untouched,
-    /// BIP69 / Lexicographic
-    Bip69Lexicographic,
+    /// Provide custom comparison functions for sorting
+    Custom {
+        /// Transaction inputs sort function
+        input_sort: Arc<TxSort<TxIn>>,
+        /// Transaction outputs sort function
+        output_sort: Arc<TxSort<TxOut>>,
+    },
+}
+
+impl core::fmt::Debug for TxOrdering {
+    fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+        match self {
+            TxOrdering::Shuffle => write!(f, "Shuffle"),
+            TxOrdering::Untouched => write!(f, "Untouched"),
+            TxOrdering::Custom { .. } => write!(f, "Custom"),
+        }
+    }
 }
 
 impl TxOrdering {
-    /// Sort transaction inputs and outputs by [`TxOrdering`] variant
+    /// Sort transaction inputs and outputs by [`TxOrdering`] variant.
+    ///
+    /// Uses the thread-local random number generator (rng).
+    #[cfg(feature = "std")]
     pub fn sort_tx(&self, tx: &mut Transaction) {
+        self.sort_tx_with_aux_rand(tx, &mut bitcoin::key::rand::thread_rng())
+    }
+
+    /// Sort transaction inputs and outputs by [`TxOrdering`] variant.
+    ///
+    /// Uses a provided random number generator (rng).
+    pub fn sort_tx_with_aux_rand(&self, tx: &mut Transaction, rng: &mut impl RngCore) {
         match self {
             TxOrdering::Untouched => {}
             TxOrdering::Shuffle => {
-                use rand::seq::SliceRandom;
-                let mut rng = rand::thread_rng();
-                tx.input.shuffle(&mut rng);
-                tx.output.shuffle(&mut rng);
+                shuffle_slice(&mut tx.input, rng);
+                shuffle_slice(&mut tx.output, rng);
             }
-            TxOrdering::Bip69Lexicographic => {
-                tx.input.sort_unstable_by_key(|txin| {
-                    (txin.previous_output.txid, txin.previous_output.vout)
-                });
-                tx.output
-                    .sort_unstable_by_key(|txout| (txout.value, txout.script_pubkey.clone()));
+            TxOrdering::Custom {
+                input_sort,
+                output_sort,
+            } => {
+                tx.input.sort_unstable_by(|a, b| input_sort(a, b));
+                tx.output.sort_unstable_by(|a, b| output_sort(a, b));
             }
         }
     }
@@ -930,15 +894,10 @@ mod test {
 
     use bdk_chain::ConfirmationTime;
     use bitcoin::consensus::deserialize;
-    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::hex::FromHex;
+    use bitcoin::TxOut;
 
     use super::*;
-
-    #[test]
-    fn test_output_ordering_default_shuffle() {
-        assert_eq!(TxOrdering::default(), TxOrdering::Shuffle);
-    }
-
     #[test]
     fn test_output_ordering_untouched() {
         let original_tx = ordering_test_tx!();
@@ -971,13 +930,28 @@ mod test {
     }
 
     #[test]
-    fn test_output_ordering_bip69() {
+    fn test_output_ordering_custom_but_bip69() {
         use core::str::FromStr;
 
         let original_tx = ordering_test_tx!();
         let mut tx = original_tx;
 
-        TxOrdering::Bip69Lexicographic.sort_tx(&mut tx);
+        let bip69_txin_cmp = |tx_a: &TxIn, tx_b: &TxIn| {
+            let project_outpoint = |t: &TxIn| (t.previous_output.txid, t.previous_output.vout);
+            project_outpoint(tx_a).cmp(&project_outpoint(tx_b))
+        };
+
+        let bip69_txout_cmp = |tx_a: &TxOut, tx_b: &TxOut| {
+            let project_utxo = |t: &TxOut| (t.value, t.script_pubkey.clone());
+            project_utxo(tx_a).cmp(&project_utxo(tx_b))
+        };
+
+        let custom_bip69_ordering = TxOrdering::Custom {
+            input_sort: Arc::new(bip69_txin_cmp),
+            output_sort: Arc::new(bip69_txout_cmp),
+        };
+
+        custom_bip69_ordering.sort_tx(&mut tx);
 
         assert_eq!(
             tx.input[0].previous_output,
@@ -1001,7 +975,7 @@ mod test {
             .unwrap()
         );
 
-        assert_eq!(tx.output[0].value, 800);
+        assert_eq!(tx.output[0].value.to_sat(), 800);
         assert_eq!(tx.output[1].script_pubkey, ScriptBuf::from(vec![0xAA]));
         assert_eq!(
             tx.output[2].script_pubkey,
@@ -1009,6 +983,63 @@ mod test {
         );
     }
 
+    #[test]
+    fn test_output_ordering_custom_with_sha256() {
+        use bitcoin::hashes::{sha256, Hash};
+
+        let original_tx = ordering_test_tx!();
+        let mut tx_1 = original_tx.clone();
+        let mut tx_2 = original_tx.clone();
+        let shared_secret = "secret_tweak";
+
+        let hash_txin_with_shared_secret_seed = Arc::new(|tx_a: &TxIn, tx_b: &TxIn| {
+            let secret_digest_from_txin = |txin: &TxIn| {
+                sha256::Hash::hash(
+                    &[
+                        &txin.previous_output.txid.to_raw_hash()[..],
+                        &txin.previous_output.vout.to_be_bytes(),
+                        shared_secret.as_bytes(),
+                    ]
+                    .concat(),
+                )
+            };
+            secret_digest_from_txin(tx_a).cmp(&secret_digest_from_txin(tx_b))
+        });
+
+        let hash_txout_with_shared_secret_seed = Arc::new(|tx_a: &TxOut, tx_b: &TxOut| {
+            let secret_digest_from_txout = |txin: &TxOut| {
+                sha256::Hash::hash(
+                    &[
+                        &txin.value.to_sat().to_be_bytes(),
+                        &txin.script_pubkey.clone().into_bytes()[..],
+                        shared_secret.as_bytes(),
+                    ]
+                    .concat(),
+                )
+            };
+            secret_digest_from_txout(tx_a).cmp(&secret_digest_from_txout(tx_b))
+        });
+
+        let custom_ordering_from_salted_sha256_1 = TxOrdering::Custom {
+            input_sort: hash_txin_with_shared_secret_seed.clone(),
+            output_sort: hash_txout_with_shared_secret_seed.clone(),
+        };
+
+        let custom_ordering_from_salted_sha256_2 = TxOrdering::Custom {
+            input_sort: hash_txin_with_shared_secret_seed,
+            output_sort: hash_txout_with_shared_secret_seed,
+        };
+
+        custom_ordering_from_salted_sha256_1.sort_tx(&mut tx_1);
+        custom_ordering_from_salted_sha256_2.sort_tx(&mut tx_2);
+
+        // Check the ordering is consistent between calls
+        assert_eq!(tx_1, tx_2);
+        // Check transaction order has changed
+        assert_ne!(tx_1, original_tx);
+        assert_ne!(tx_2, original_tx);
+    }
+
     fn get_test_utxos() -> Vec<LocalOutput> {
         use bitcoin::hashes::Hash;
 
@@ -1018,7 +1049,7 @@ mod test {
                     txid: bitcoin::Txid::from_slice(&[0; 32]).unwrap(),
                     vout: 0,
                 },
-                txout: Default::default(),
+                txout: TxOut::NULL,
                 keychain: KeychainKind::External,
                 is_spent: false,
                 confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
@@ -1029,7 +1060,7 @@ mod test {
                     txid: bitcoin::Txid::from_slice(&[0; 32]).unwrap(),
                     vout: 1,
                 },
-                txout: Default::default(),
+                txout: TxOut::NULL,
                 keychain: KeychainKind::Internal,
                 is_spent: false,
                 confirmation_time: ConfirmationTime::Confirmed {

crates/wallet/src/wallet/utils.rs

@@ -10,10 +10,12 @@
 // licenses.
 
 use bitcoin::secp256k1::{All, Secp256k1};
-use bitcoin::{absolute, Script, Sequence};
+use bitcoin::{absolute, relative, Script, Sequence};
 
 use miniscript::{MiniscriptKey, Satisfier, ToPublicKey};
 
+use rand_core::RngCore;
+
 /// Trait to check if a value is below the dust limit.
 /// We are performing dust value calculation for a given script public key using rust-bitcoin to
 /// keep it compatible with network dust rate
@@ -26,7 +28,7 @@ pub trait IsDust {
 
 impl IsDust for u64 {
     fn is_dust(&self, script: &Script) -> bool {
-        *self < script.dust_value().to_sat()
+        *self < script.minimal_non_dust().to_sat()
     }
 }
 
@@ -95,7 +97,7 @@ impl Older {
 }
 
 impl<Pk: MiniscriptKey + ToPublicKey> Satisfier<Pk> for Older {
-    fn check_older(&self, n: Sequence) -> bool {
+    fn check_older(&self, n: relative::LockTime) -> bool {
         if let Some(current_height) = self.current_height {
             // TODO: test >= / >
             current_height
@@ -110,6 +112,19 @@ impl<Pk: MiniscriptKey + ToPublicKey> Satisfier<Pk> for Older {
     }
 }
 
+// The Knuth shuffling algorithm based on the original [Fisher-Yates method](https://en.wikipedia.org/wiki/Fisher%E2%80%93Yates_shuffle)
+pub(crate) fn shuffle_slice<T>(list: &mut [T], rng: &mut impl RngCore) {
+    if list.is_empty() {
+        return;
+    }
+    let mut current_index = list.len() - 1;
+    while current_index > 0 {
+        let random_index = rng.next_u32() as usize % (current_index + 1);
+        list.swap(current_index, random_index);
+        current_index -= 1;
+    }
+}
+
 pub(crate) type SecpCtx = Secp256k1<All>;
 
 #[cfg(test)]
@@ -118,9 +133,11 @@ mod test {
     // otherwise it's time-based
     pub(crate) const SEQUENCE_LOCKTIME_TYPE_FLAG: u32 = 1 << 22;
 
-    use super::{check_nsequence_rbf, IsDust};
+    use super::{check_nsequence_rbf, shuffle_slice, IsDust};
     use crate::bitcoin::{Address, Network, Sequence};
+    use alloc::vec::Vec;
     use core::str::FromStr;
+    use rand::{rngs::StdRng, thread_rng, SeedableRng};
 
     #[test]
     fn test_is_dust() {
@@ -138,7 +155,7 @@ mod test {
             .require_network(Network::Bitcoin)
             .unwrap()
             .script_pubkey();
-        assert!(script_p2wpkh.is_v0_p2wpkh());
+        assert!(script_p2wpkh.is_p2wpkh());
         assert!(293.is_dust(&script_p2wpkh));
         assert!(!294.is_dust(&script_p2wpkh));
     }
@@ -182,4 +199,46 @@ mod test {
         );
         assert!(result);
     }
+
+    #[test]
+    #[cfg(feature = "std")]
+    fn test_shuffle_slice_empty_vec() {
+        let mut test: Vec<u8> = vec![];
+        shuffle_slice(&mut test, &mut thread_rng());
+    }
+
+    #[test]
+    #[cfg(feature = "std")]
+    fn test_shuffle_slice_single_vec() {
+        let mut test: Vec<u8> = vec![0];
+        shuffle_slice(&mut test, &mut thread_rng());
+    }
+
+    #[test]
+    fn test_shuffle_slice_duple_vec() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut test: Vec<u8> = vec![0, 1];
+        shuffle_slice(&mut test, &mut rng);
+        assert_eq!(test, &[0, 1]);
+        let seed = [6; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut test: Vec<u8> = vec![0, 1];
+        shuffle_slice(&mut test, &mut rng);
+        assert_eq!(test, &[1, 0]);
+    }
+
+    #[test]
+    fn test_shuffle_slice_multi_vec() {
+        let seed = [0; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut test: Vec<u8> = vec![0, 1, 2, 4, 5];
+        shuffle_slice(&mut test, &mut rng);
+        assert_eq!(test, &[2, 1, 0, 4, 5]);
+        let seed = [25; 32];
+        let mut rng: StdRng = SeedableRng::from_seed(seed);
+        let mut test: Vec<u8> = vec![0, 1, 2, 4, 5];
+        shuffle_slice(&mut test, &mut rng);
+        assert_eq!(test, &[0, 4, 1, 2, 5]);
+    }
 }

crates/wallet/tests/common.rs

@@ -0,0 +1,230 @@
+#![allow(unused)]
+use bdk_chain::{BlockId, ConfirmationBlockTime, ConfirmationTime, TxGraph};
+use bdk_wallet::{CreateParams, KeychainKind, LocalOutput, Update, Wallet};
+use bitcoin::{
+    hashes::Hash, transaction, Address, Amount, BlockHash, FeeRate, Network, OutPoint, Transaction,
+    TxIn, TxOut, Txid,
+};
+use std::str::FromStr;
+
+/// Return a fake wallet that appears to be funded for testing.
+///
+/// The funded wallet contains a tx with a 76_000 sats input and two outputs, one spending 25_000
+/// to a foreign address and one returning 50_000 back to the wallet. The remaining 1000
+/// sats are the transaction fee.
+pub fn get_funded_wallet_with_change(descriptor: &str, change: &str) -> (Wallet, bitcoin::Txid) {
+    let mut wallet = Wallet::create(descriptor.to_string(), change.to_string())
+        .network(Network::Regtest)
+        .create_wallet_no_persist()
+        .expect("descriptors must be valid");
+
+    let receive_address = wallet.peek_address(KeychainKind::External, 0).address;
+    let sendto_address = Address::from_str("bcrt1q3qtze4ys45tgdvguj66zrk4fu6hq3a3v9pfly5")
+        .expect("address")
+        .require_network(Network::Regtest)
+        .unwrap();
+
+    let tx0 = Transaction {
+        version: transaction::Version::ONE,
+        lock_time: bitcoin::absolute::LockTime::ZERO,
+        input: vec![TxIn {
+            previous_output: OutPoint {
+                txid: Txid::all_zeros(),
+                vout: 0,
+            },
+            script_sig: Default::default(),
+            sequence: Default::default(),
+            witness: Default::default(),
+        }],
+        output: vec![TxOut {
+            value: Amount::from_sat(76_000),
+            script_pubkey: receive_address.script_pubkey(),
+        }],
+    };
+
+    let tx1 = Transaction {
+        version: transaction::Version::ONE,
+        lock_time: bitcoin::absolute::LockTime::ZERO,
+        input: vec![TxIn {
+            previous_output: OutPoint {
+                txid: tx0.compute_txid(),
+                vout: 0,
+            },
+            script_sig: Default::default(),
+            sequence: Default::default(),
+            witness: Default::default(),
+        }],
+        output: vec![
+            TxOut {
+                value: Amount::from_sat(50_000),
+                script_pubkey: receive_address.script_pubkey(),
+            },
+            TxOut {
+                value: Amount::from_sat(25_000),
+                script_pubkey: sendto_address.script_pubkey(),
+            },
+        ],
+    };
+
+    wallet
+        .insert_checkpoint(BlockId {
+            height: 42,
+            hash: BlockHash::all_zeros(),
+        })
+        .unwrap();
+    wallet
+        .insert_checkpoint(BlockId {
+            height: 1_000,
+            hash: BlockHash::all_zeros(),
+        })
+        .unwrap();
+    wallet
+        .insert_checkpoint(BlockId {
+            height: 2_000,
+            hash: BlockHash::all_zeros(),
+        })
+        .unwrap();
+
+    wallet.insert_tx(tx0.clone());
+    insert_anchor_from_conf(
+        &mut wallet,
+        tx0.compute_txid(),
+        ConfirmationTime::Confirmed {
+            height: 1_000,
+            time: 100,
+        },
+    );
+
+    wallet.insert_tx(tx1.clone());
+    insert_anchor_from_conf(
+        &mut wallet,
+        tx1.compute_txid(),
+        ConfirmationTime::Confirmed {
+            height: 2_000,
+            time: 200,
+        },
+    );
+
+    (wallet, tx1.compute_txid())
+}
+
+/// Return a fake wallet that appears to be funded for testing.
+///
+/// The funded wallet contains a tx with a 76_000 sats input and two outputs, one spending 25_000
+/// to a foreign address and one returning 50_000 back to the wallet. The remaining 1000
+/// sats are the transaction fee.
+///
+/// Note: the change descriptor will have script type `p2wpkh`. If passing some other script type
+/// as argument, make sure you're ok with getting a wallet where the keychains have potentially
+/// different script types. Otherwise, use `get_funded_wallet_with_change`.
+pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
+    let change = get_test_wpkh_change();
+    get_funded_wallet_with_change(descriptor, change)
+}
+
+pub fn get_funded_wallet_wpkh() -> (Wallet, bitcoin::Txid) {
+    get_funded_wallet_with_change(get_test_wpkh(), get_test_wpkh_change())
+}
+
+pub fn get_test_wpkh() -> &'static str {
+    "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)"
+}
+
+pub fn get_test_wpkh_with_change_desc() -> (&'static str, &'static str) {
+    (
+        "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)",
+        get_test_wpkh_change(),
+    )
+}
+
+fn get_test_wpkh_change() -> &'static str {
+    "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/0)"
+}
+
+pub fn get_test_single_sig_csv() -> &'static str {
+    // and(pk(Alice),older(6))
+    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),older(6)))"
+}
+
+pub fn get_test_a_or_b_plus_csv() -> &'static str {
+    // or(pk(Alice),and(pk(Bob),older(144)))
+    "wsh(or_d(pk(cRjo6jqfVNP33HhSS76UhXETZsGTZYx8FMFvR9kpbtCSV1PmdZdu),and_v(v:pk(cMnkdebixpXMPfkcNEjjGin7s94hiehAH4mLbYkZoh9KSiNNmqC8),older(144))))"
+}
+
+pub fn get_test_single_sig_cltv() -> &'static str {
+    // and(pk(Alice),after(100000))
+    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100000)))"
+}
+
+pub fn get_test_tr_single_sig() -> &'static str {
+    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG)"
+}
+
+pub fn get_test_tr_with_taptree() -> &'static str {
+    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
+}
+
+pub fn get_test_tr_with_taptree_both_priv() -> &'static str {
+    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(cNaQCDwmmh4dS9LzCgVtyy1e1xjCJ21GUDHe9K98nzb689JvinGV)})"
+}
+
+pub fn get_test_tr_repeated_key() -> &'static str {
+    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100)),and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(200))})"
+}
+
+pub fn get_test_tr_single_sig_xprv() -> &'static str {
+    "tr(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*)"
+}
+
+pub fn get_test_tr_single_sig_xprv_with_change_desc() -> (&'static str, &'static str) {
+    ("tr(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/0/*)",
+    "tr(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/1/*)")
+}
+
+pub fn get_test_tr_with_taptree_xprv() -> &'static str {
+    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
+}
+
+pub fn get_test_tr_dup_keys() -> &'static str {
+    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
+}
+
+/// Construct a new [`FeeRate`] from the given raw `sat_vb` feerate. This is
+/// useful in cases where we want to create a feerate from a `f64`, as the
+/// traditional [`FeeRate::from_sat_per_vb`] method will only accept an integer.
+///
+/// **Note** this 'quick and dirty' conversion should only be used when the input
+/// parameter has units of `satoshis/vbyte` **AND** is not expected to overflow,
+/// or else the resulting value will be inaccurate.
+pub fn feerate_unchecked(sat_vb: f64) -> FeeRate {
+    // 1 sat_vb / 4wu_vb * 1000kwu_wu = 250 sat_kwu
+    let sat_kwu = (sat_vb * 250.0).ceil() as u64;
+    FeeRate::from_sat_per_kwu(sat_kwu)
+}
+
+/// Simulates confirming a tx with `txid` at the specified `position` by inserting an anchor
+/// at the lowest height in local chain that is greater or equal to `position`'s height,
+/// assuming the confirmation time matches `ConfirmationTime::Confirmed`.
+pub fn insert_anchor_from_conf(wallet: &mut Wallet, txid: Txid, position: ConfirmationTime) {
+    if let ConfirmationTime::Confirmed { height, time } = position {
+        // anchor tx to checkpoint with lowest height that is >= position's height
+        let anchor = wallet
+            .local_chain()
+            .range(height..)
+            .last()
+            .map(|anchor_cp| ConfirmationBlockTime {
+                block_id: anchor_cp.block_id(),
+                confirmation_time: time,
+            })
+            .expect("confirmation height cannot be greater than tip");
+
+        let mut graph = TxGraph::default();
+        let _ = graph.insert_anchor(txid, anchor);
+        wallet
+            .apply_update(Update {
+                graph,
+                ..Default::default()
+            })
+            .unwrap();
+    }
+}

crates/wallet/tests/psbt.rs

@@ -1,8 +1,5 @@
-use bdk::bitcoin::TxIn;
-use bdk::wallet::AddressIndex;
-use bdk::wallet::AddressIndex::New;
-use bdk::{psbt, FeeRate, SignOptions};
-use bitcoin::psbt::PartiallySignedTransaction as Psbt;
+use bdk_wallet::bitcoin::{Amount, FeeRate, Psbt, TxIn};
+use bdk_wallet::{psbt, KeychainKind, SignOptions};
 use core::str::FromStr;
 mod common;
 use common::*;
@@ -15,9 +12,9 @@ const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6
 fn test_psbt_malformed_psbt_input_legacy() {
     let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
+    let send_to = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    builder.add_recipient(send_to.script_pubkey(), Amount::from_sat(10_000));
     let mut psbt = builder.finish().unwrap();
     psbt.inputs.push(psbt_bip.inputs[0].clone());
     let options = SignOptions {
@@ -32,9 +29,9 @@ fn test_psbt_malformed_psbt_input_legacy() {
 fn test_psbt_malformed_psbt_input_segwit() {
     let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
+    let send_to = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    builder.add_recipient(send_to.script_pubkey(), Amount::from_sat(10_000));
     let mut psbt = builder.finish().unwrap();
     psbt.inputs.push(psbt_bip.inputs[1].clone());
     let options = SignOptions {
@@ -48,9 +45,9 @@ fn test_psbt_malformed_psbt_input_segwit() {
 #[should_panic(expected = "InputIndexOutOfRange")]
 fn test_psbt_malformed_tx_input() {
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
+    let send_to = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    builder.add_recipient(send_to.script_pubkey(), Amount::from_sat(10_000));
     let mut psbt = builder.finish().unwrap();
     psbt.unsigned_tx.input.push(TxIn::default());
     let options = SignOptions {
@@ -64,9 +61,9 @@ fn test_psbt_malformed_tx_input() {
 fn test_psbt_sign_with_finalized() {
     let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
     let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
+    let send_to = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    builder.add_recipient(send_to.script_pubkey(), Amount::from_sat(10_000));
     let mut psbt = builder.finish().unwrap();
 
     // add a finalized input
@@ -82,13 +79,13 @@ fn test_psbt_sign_with_finalized() {
 fn test_psbt_fee_rate_with_witness_utxo() {
     use psbt::PsbtUtils;
 
-    let expected_fee_rate = 1.2345;
+    let expected_fee_rate = FeeRate::from_sat_per_kwu(310);
 
     let (mut wallet, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wallet.get_address(New);
+    let addr = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
+    builder.fee_rate(expected_fee_rate);
     let mut psbt = builder.finish().unwrap();
     let fee_amount = psbt.fee_amount();
     assert!(fee_amount.is_some());
@@ -99,21 +96,21 @@ fn test_psbt_fee_rate_with_witness_utxo() {
     assert!(finalized);
 
     let finalized_fee_rate = psbt.fee_rate().unwrap();
-    assert!(finalized_fee_rate.as_sat_per_vb() >= expected_fee_rate);
-    assert!(finalized_fee_rate.as_sat_per_vb() < unfinalized_fee_rate.as_sat_per_vb());
+    assert!(finalized_fee_rate >= expected_fee_rate);
+    assert!(finalized_fee_rate < unfinalized_fee_rate);
 }
 
 #[test]
 fn test_psbt_fee_rate_with_nonwitness_utxo() {
     use psbt::PsbtUtils;
 
-    let expected_fee_rate = 1.2345;
+    let expected_fee_rate = FeeRate::from_sat_per_kwu(310);
 
     let (mut wallet, _) = get_funded_wallet("pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wallet.get_address(New);
+    let addr = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
+    builder.fee_rate(expected_fee_rate);
     let mut psbt = builder.finish().unwrap();
     let fee_amount = psbt.fee_amount();
     assert!(fee_amount.is_some());
@@ -123,21 +120,21 @@ fn test_psbt_fee_rate_with_nonwitness_utxo() {
     assert!(finalized);
 
     let finalized_fee_rate = psbt.fee_rate().unwrap();
-    assert!(finalized_fee_rate.as_sat_per_vb() >= expected_fee_rate);
-    assert!(finalized_fee_rate.as_sat_per_vb() < unfinalized_fee_rate.as_sat_per_vb());
+    assert!(finalized_fee_rate >= expected_fee_rate);
+    assert!(finalized_fee_rate < unfinalized_fee_rate);
 }
 
 #[test]
 fn test_psbt_fee_rate_with_missing_txout() {
     use psbt::PsbtUtils;
 
-    let expected_fee_rate = 1.2345;
+    let expected_fee_rate = FeeRate::from_sat_per_kwu(310);
 
     let (mut wpkh_wallet,  _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wpkh_wallet.get_address(New);
+    let addr = wpkh_wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wpkh_wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
+    builder.fee_rate(expected_fee_rate);
     let mut wpkh_psbt = builder.finish().unwrap();
 
     wpkh_psbt.inputs[0].witness_utxo = None;
@@ -145,11 +142,13 @@ fn test_psbt_fee_rate_with_missing_txout() {
     assert!(wpkh_psbt.fee_amount().is_none());
     assert!(wpkh_psbt.fee_rate().is_none());
 
-    let (mut pkh_wallet,  _) = get_funded_wallet("pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = pkh_wallet.get_address(New);
+    let desc = "pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/0)";
+    let change_desc = "pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/1)";
+    let (mut pkh_wallet, _) = get_funded_wallet_with_change(desc, change_desc);
+    let addr = pkh_wallet.peek_address(KeychainKind::External, 0);
     let mut builder = pkh_wallet.build_tx();
     builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
+    builder.fee_rate(expected_fee_rate);
     let mut pkh_psbt = builder.finish().unwrap();
 
     pkh_psbt.inputs[0].non_witness_utxo = None;
@@ -159,18 +158,29 @@ fn test_psbt_fee_rate_with_missing_txout() {
 
 #[test]
 fn test_psbt_multiple_internalkey_signers() {
-    use bdk::signer::{SignerContext, SignerOrdering, SignerWrapper};
-    use bdk::KeychainKind;
-    use bitcoin::{secp256k1::Secp256k1, PrivateKey};
-    use miniscript::psbt::PsbtExt;
+    use bdk_wallet::signer::{SignerContext, SignerOrdering, SignerWrapper};
+    use bdk_wallet::KeychainKind;
+    use bitcoin::key::TapTweak;
+    use bitcoin::secp256k1::{schnorr, Keypair, Message, Secp256k1, XOnlyPublicKey};
+    use bitcoin::sighash::{Prevouts, SighashCache, TapSighashType};
+    use bitcoin::{PrivateKey, TxOut};
     use std::sync::Arc;
 
     let secp = Secp256k1::new();
-    let (mut wallet, _) = get_funded_wallet(get_test_tr_single_sig());
-    let send_to = wallet.get_address(AddressIndex::New);
+    let wif = "cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG";
+    let desc = format!("tr({})", wif);
+    let prv = PrivateKey::from_wif(wif).unwrap();
+    let keypair = Keypair::from_secret_key(&secp, &prv.inner);
+
+    let change_desc = "tr(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)";
+    let (mut wallet, _) = get_funded_wallet_with_change(&desc, change_desc);
+    let to_spend = wallet.balance().total();
+    let send_to = wallet.peek_address(KeychainKind::External, 0);
     let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
+    builder.drain_to(send_to.script_pubkey()).drain_wallet();
     let mut psbt = builder.finish().unwrap();
+    let unsigned_tx = psbt.unsigned_tx.clone();
+
     // Adds a signer for the wrong internal key, bdk should not use this key to sign
     wallet.add_signer(
         KeychainKind::External,
@@ -183,10 +193,32 @@ fn test_psbt_multiple_internalkey_signers() {
             },
         )),
     );
-    let _ = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
-    // Checks that we signed using the right key
-    assert!(
-        psbt.finalize_mut(&secp).is_ok(),
-        "The wrong internal key was used"
-    );
+    let finalized = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
+    assert!(finalized);
+
+    // To verify, we need the signature, message, and pubkey
+    let witness = psbt.inputs[0].final_script_witness.as_ref().unwrap();
+    assert!(!witness.is_empty());
+    let signature = schnorr::Signature::from_slice(witness.iter().next().unwrap()).unwrap();
+
+    // the prevout we're spending
+    let prevouts = &[TxOut {
+        script_pubkey: send_to.script_pubkey(),
+        value: to_spend,
+    }];
+    let prevouts = Prevouts::All(prevouts);
+    let input_index = 0;
+    let mut sighash_cache = SighashCache::new(unsigned_tx);
+    let sighash = sighash_cache
+        .taproot_key_spend_signature_hash(input_index, &prevouts, TapSighashType::Default)
+        .unwrap();
+    let message = Message::from(sighash);
+
+    // add tweak. this was taken from `signer::sign_psbt_schnorr`
+    let keypair = keypair.tap_tweak(&secp, None).to_inner();
+    let (xonlykey, _parity) = XOnlyPublicKey::from_keypair(&keypair);
+
+    // Must verify if we used the correct key to sign
+    let verify_res = secp.verify_schnorr(&signature, &message, &xonlykey);
+    assert!(verify_res.is_ok(), "The wrong internal key was used");
 }

example-crates/example_bitcoind_rpc_polling/src/main.rs

@@ -13,9 +13,10 @@ use bdk_bitcoind_rpc::{
 };
 use bdk_chain::{
     bitcoin::{constants::genesis_block, Block, Transaction},
-    indexed_tx_graph, keychain,
+    indexed_tx_graph,
+    indexer::keychain_txout,
     local_chain::{self, LocalChain},
-    ConfirmationTimeHeightAnchor, IndexedTxGraph,
+    ConfirmationBlockTime, IndexedTxGraph, Merge,
 };
 use example_cli::{
     anyhow,
@@ -37,7 +38,7 @@ const DB_COMMIT_DELAY: Duration = Duration::from_secs(60);
 
 type ChangeSet = (
     local_chain::ChangeSet,
-    indexed_tx_graph::ChangeSet<ConfirmationTimeHeightAnchor, keychain::ChangeSet<Keychain>>,
+    indexed_tx_graph::ChangeSet<ConfirmationBlockTime, keychain_txout::ChangeSet>,
 );
 
 #[derive(Debug)]
@@ -137,8 +138,7 @@ fn main() -> anyhow::Result<()> {
         let genesis_hash = genesis_block(args.network).block_hash();
         let (chain, chain_changeset) = LocalChain::from_genesis_hash(genesis_hash);
         let mut db = db.lock().unwrap();
-        db.stage((chain_changeset, Default::default()));
-        db.commit()?;
+        db.append_changeset(&(chain_changeset, Default::default()))?;
         chain
     } else {
         LocalChain::from_changeset(init_chain_changeset)?
@@ -176,6 +176,7 @@ fn main() -> anyhow::Result<()> {
             let chain_tip = chain.lock().unwrap().tip();
             let rpc_client = rpc_args.new_client()?;
             let mut emitter = Emitter::new(&rpc_client, chain_tip, fallback_height);
+            let mut db_stage = ChangeSet::default();
 
             let mut last_db_commit = Instant::now();
             let mut last_print = Instant::now();
@@ -185,21 +186,20 @@ fn main() -> anyhow::Result<()> {
 
                 let mut chain = chain.lock().unwrap();
                 let mut graph = graph.lock().unwrap();
-                let mut db = db.lock().unwrap();
 
                 let chain_changeset = chain
-                    .apply_update(local_chain::Update {
-                        tip: emission.checkpoint,
-                        introduce_older_blocks: false,
-                    })
+                    .apply_update(emission.checkpoint)
                     .expect("must always apply as we receive blocks in order from emitter");
                 let graph_changeset = graph.apply_block_relevant(&emission.block, height);
-                db.stage((chain_changeset, graph_changeset));
+                db_stage.merge((chain_changeset, graph_changeset));
 
                 // commit staged db changes in intervals
                 if last_db_commit.elapsed() >= DB_COMMIT_DELAY {
+                    let db = &mut *db.lock().unwrap();
                     last_db_commit = Instant::now();
-                    db.commit()?;
+                    if let Some(changeset) = db_stage.take() {
+                        db.append_changeset(&changeset)?;
+                    }
                     println!(
                         "[{:>10}s] committed to db (took {}s)",
                         start.elapsed().as_secs_f32(),
@@ -234,9 +234,11 @@ fn main() -> anyhow::Result<()> {
                 mempool_txs.iter().map(|(tx, time)| (tx, *time)),
             );
             {
-                let mut db = db.lock().unwrap();
-                db.stage((local_chain::ChangeSet::default(), graph_changeset));
-                db.commit()?; // commit one last time
+                let db = &mut *db.lock().unwrap();
+                db_stage.merge((local_chain::ChangeSet::default(), graph_changeset));
+                if let Some(changeset) = db_stage.take() {
+                    db.append_changeset(&changeset)?;
+                }
             }
         }
         RpcCommands::Live { rpc_args } => {
@@ -292,21 +294,17 @@ fn main() -> anyhow::Result<()> {
             let mut tip_height = 0_u32;
             let mut last_db_commit = Instant::now();
             let mut last_print = Option::<Instant>::None;
+            let mut db_stage = ChangeSet::default();
 
             for emission in rx {
-                let mut db = db.lock().unwrap();
                 let mut graph = graph.lock().unwrap();
                 let mut chain = chain.lock().unwrap();
 
                 let changeset = match emission {
                     Emission::Block(block_emission) => {
                         let height = block_emission.block_height();
-                        let chain_update = local_chain::Update {
-                            tip: block_emission.checkpoint,
-                            introduce_older_blocks: false,
-                        };
                         let chain_changeset = chain
-                            .apply_update(chain_update)
+                            .apply_update(block_emission.checkpoint)
                             .expect("must always apply as we receive blocks in order from emitter");
                         let graph_changeset =
                             graph.apply_block_relevant(&block_emission.block, height);
@@ -323,12 +321,14 @@ fn main() -> anyhow::Result<()> {
                         continue;
                     }
                 };
-
-                db.stage(changeset);
+                db_stage.merge(changeset);
 
                 if last_db_commit.elapsed() >= DB_COMMIT_DELAY {
+                    let db = &mut *db.lock().unwrap();
                     last_db_commit = Instant::now();
-                    db.commit()?;
+                    if let Some(changeset) = db_stage.take() {
+                        db.append_changeset(&changeset)?;
+                    }
                     println!(
                         "[{:>10}s] committed to db (took {}s)",
                         start.elapsed().as_secs_f32(),

example-crates/example_cli/src/lib.rs

@@ -3,21 +3,24 @@ use anyhow::Context;
 use bdk_coin_select::{coin_select_bnb, CoinSelector, CoinSelectorOpt, WeightedValue};
 use bdk_file_store::Store;
 use serde::{de::DeserializeOwned, Serialize};
-use std::{cmp::Reverse, collections::HashMap, path::PathBuf, sync::Mutex, time::Duration};
+use std::fmt::Debug;
+use std::{cmp::Reverse, collections::BTreeMap, path::PathBuf, sync::Mutex, time::Duration};
 
 use bdk_chain::{
     bitcoin::{
-        absolute, address, psbt::Prevouts, secp256k1::Secp256k1, sighash::SighashCache, Address,
-        Network, Sequence, Transaction, TxIn, TxOut,
+        absolute, address,
+        secp256k1::Secp256k1,
+        sighash::{Prevouts, SighashCache},
+        transaction, Address, Amount, Network, Sequence, Transaction, TxIn, TxOut,
     },
     indexed_tx_graph::{self, IndexedTxGraph},
-    keychain::{self, KeychainTxOutIndex},
+    indexer::keychain_txout::{self, KeychainTxOutIndex},
     local_chain,
     miniscript::{
         descriptor::{DescriptorSecretKey, KeyMap},
         Descriptor, DescriptorPublicKey,
     },
-    Anchor, Append, ChainOracle, DescriptorExt, FullTxOut, Persist, PersistBackend,
+    Anchor, ChainOracle, DescriptorExt, FullTxOut, Merge,
 };
 pub use bdk_file_store;
 pub use clap;
@@ -27,9 +30,8 @@ use clap::{Parser, Subcommand};
 pub type KeychainTxGraph<A> = IndexedTxGraph<A, KeychainTxOutIndex<Keychain>>;
 pub type KeychainChangeSet<A> = (
     local_chain::ChangeSet,
-    indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<Keychain>>,
+    indexed_tx_graph::ChangeSet<A, keychain_txout::ChangeSet>,
 );
-pub type Database<C> = Persist<Store<C>, C>;
 
 #[derive(Parser)]
 #[clap(author, version, about, long_about = None)]
@@ -189,7 +191,7 @@ impl core::fmt::Display for Keychain {
 }
 
 pub struct CreateTxChange {
-    pub index_changeset: keychain::ChangeSet<Keychain>,
+    pub index_changeset: keychain_txout::ChangeSet,
     pub change_keychain: Keychain,
     pub index: u32,
 }
@@ -197,7 +199,7 @@ pub struct CreateTxChange {
 pub fn create_tx<A: Anchor, O: ChainOracle>(
     graph: &mut KeychainTxGraph<A>,
     chain: &O,
-    keymap: &HashMap<DescriptorPublicKey, DescriptorSecretKey>,
+    keymap: &BTreeMap<DescriptorPublicKey, DescriptorSecretKey>,
     cs_algorithm: CoinSelectionAlgo,
     address: Address,
     value: u64,
@@ -205,7 +207,7 @@ pub fn create_tx<A: Anchor, O: ChainOracle>(
 where
     O::Error: std::error::Error + Send + Sync + 'static,
 {
-    let mut changeset = keychain::ChangeSet::default();
+    let mut changeset = keychain_txout::ChangeSet::default();
 
     let assets = bdk_tmp_plan::Assets {
         keys: keymap.iter().map(|(pk, _)| pk.clone()).collect(),
@@ -235,7 +237,7 @@ where
         .iter()
         .map(|(plan, utxo)| {
             WeightedValue::new(
-                utxo.txout.value,
+                utxo.txout.value.to_sat(),
                 plan.expected_weight() as _,
                 plan.witness_version().is_some(),
             )
@@ -243,29 +245,33 @@ where
         .collect();
 
     let mut outputs = vec![TxOut {
-        value,
+        value: Amount::from_sat(value),
         script_pubkey: address.script_pubkey(),
     }];
 
-    let internal_keychain = if graph.index.keychains().get(&Keychain::Internal).is_some() {
+    let internal_keychain = if graph
+        .index
+        .keychains()
+        .any(|(k, _)| k == Keychain::Internal)
+    {
         Keychain::Internal
     } else {
         Keychain::External
     };
 
-    let ((change_index, change_script), change_changeset) =
-        graph.index.next_unused_spk(&internal_keychain);
-    changeset.append(change_changeset);
-
-    // Clone to drop the immutable reference.
-    let change_script = change_script.into();
+    let ((change_index, change_script), change_changeset) = graph
+        .index
+        .next_unused_spk(internal_keychain)
+        .expect("Must exist");
+    changeset.merge(change_changeset);
 
     let change_plan = bdk_tmp_plan::plan_satisfaction(
         &graph
             .index
             .keychains()
-            .get(&internal_keychain)
+            .find(|(k, _)| *k == internal_keychain)
             .expect("must exist")
+            .1
             .at_derivation_index(change_index)
             .expect("change_index can't be hardened"),
         &assets,
@@ -273,7 +279,7 @@ where
     .expect("failed to obtain change plan");
 
     let mut change_output = TxOut {
-        value: 0,
+        value: Amount::ZERO,
         script_pubkey: change_script,
     };
 
@@ -282,8 +288,9 @@ where
         min_drain_value: graph
             .index
             .keychains()
-            .get(&internal_keychain)
+            .find(|(k, _)| *k == internal_keychain)
             .expect("must exist")
+            .1
             .dust_value(),
         ..CoinSelectorOpt::fund_outputs(
             &outputs,
@@ -311,13 +318,13 @@ where
     let selected_txos = selection.apply_selection(&candidates).collect::<Vec<_>>();
 
     if let Some(drain_value) = selection_meta.drain_value {
-        change_output.value = drain_value;
+        change_output.value = Amount::from_sat(drain_value);
         // if the selection tells us to use change and the change value is sufficient, we add it as an output
         outputs.push(change_output)
     }
 
     let mut transaction = Transaction {
-        version: 0x02,
+        version: transaction::Version::TWO,
         // because the temporary planning module does not support timelocks, we can use the chain
         // tip as the `lock_time` for anti-fee-sniping purposes
         lock_time: absolute::LockTime::from_height(chain.get_chain_tip()?.height)
@@ -414,10 +421,10 @@ pub fn planned_utxos<A: Anchor, O: ChainOracle, K: Clone + bdk_tmp_plan::CanDeri
     assets: &bdk_tmp_plan::Assets<K>,
 ) -> Result<Vec<PlannedUtxo<K, A>>, O::Error> {
     let chain_tip = chain.get_chain_tip()?;
-    let outpoints = graph.index.outpoints().iter().cloned();
+    let outpoints = graph.index.outpoints();
     graph
         .graph()
-        .try_filter_chain_unspents(chain, chain_tip, outpoints)
+        .try_filter_chain_unspents(chain, chain_tip, outpoints.iter().cloned())
         .filter_map(|r| -> Option<Result<PlannedUtxo<K, A>, _>> {
             let (k, i, full_txo) = match r {
                 Err(err) => return Some(Err(err)),
@@ -426,8 +433,9 @@ pub fn planned_utxos<A: Anchor, O: ChainOracle, K: Clone + bdk_tmp_plan::CanDeri
             let desc = graph
                 .index
                 .keychains()
-                .get(&k)
+                .find(|(keychain, _)| *keychain == k)
                 .expect("keychain must exist")
+                .1
                 .at_derivation_index(i)
                 .expect("i can't be hardened");
             let plan = bdk_tmp_plan::plan_satisfaction(&desc, assets)?;
@@ -438,16 +446,23 @@ pub fn planned_utxos<A: Anchor, O: ChainOracle, K: Clone + bdk_tmp_plan::CanDeri
 
 pub fn handle_commands<CS: clap::Subcommand, S: clap::Args, A: Anchor, O: ChainOracle, C>(
     graph: &Mutex<KeychainTxGraph<A>>,
-    db: &Mutex<Database<C>>,
+    db: &Mutex<Store<C>>,
     chain: &Mutex<O>,
-    keymap: &HashMap<DescriptorPublicKey, DescriptorSecretKey>,
+    keymap: &BTreeMap<DescriptorPublicKey, DescriptorSecretKey>,
     network: Network,
     broadcast: impl FnOnce(S, &Transaction) -> anyhow::Result<()>,
     cmd: Commands<CS, S>,
 ) -> anyhow::Result<()>
 where
     O::Error: std::error::Error + Send + Sync + 'static,
-    C: Default + Append + DeserializeOwned + Serialize + From<KeychainChangeSet<A>>,
+    C: Default
+        + Merge
+        + DeserializeOwned
+        + Serialize
+        + From<KeychainChangeSet<A>>
+        + Send
+        + Sync
+        + Debug,
 {
     match cmd {
         Commands::ChainSpecific(_) => unreachable!("example code should handle this!"),
@@ -463,14 +478,15 @@ where
                         _ => unreachable!("only these two variants exist in match arm"),
                     };
 
-                    let ((spk_i, spk), index_changeset) = spk_chooser(index, &Keychain::External);
+                    let ((spk_i, spk), index_changeset) =
+                        spk_chooser(index, Keychain::External).expect("Must exist");
                     let db = &mut *db.lock().unwrap();
-                    db.stage_and_commit(C::from((
+                    db.append_changeset(&C::from((
                         local_chain::ChangeSet::default(),
                         indexed_tx_graph::ChangeSet::from(index_changeset),
                     )))?;
-                    let addr =
-                        Address::from_script(spk, network).context("failed to derive address")?;
+                    let addr = Address::from_script(spk.as_script(), network)
+                        .context("failed to derive address")?;
                     println!("[address @ {}] {}", spk_i, addr);
                     Ok(())
                 }
@@ -485,8 +501,8 @@ where
                         true => Keychain::Internal,
                         false => Keychain::External,
                     };
-                    for (spk_i, spk) in index.revealed_keychain_spks(&target_keychain) {
-                        let address = Address::from_script(spk, network)
+                    for (spk_i, spk) in index.revealed_keychain_spks(target_keychain) {
+                        let address = Address::from_script(spk.as_script(), network)
                             .expect("should always be able to derive address");
                         println!(
                             "{:?} {} used:{}",
@@ -504,11 +520,11 @@ where
             let chain = &*chain.lock().unwrap();
             fn print_balances<'a>(
                 title_str: &'a str,
-                items: impl IntoIterator<Item = (&'a str, u64)>,
+                items: impl IntoIterator<Item = (&'a str, Amount)>,
             ) {
                 println!("{}:", title_str);
                 for (name, amount) in items.into_iter() {
-                    println!("    {:<10} {:>12} sats", name, amount)
+                    println!("    {:<10} {:>12} sats", name, amount.to_sat())
                 }
             }
 
@@ -545,7 +561,7 @@ where
             let graph = &*graph.lock().unwrap();
             let chain = &*chain.lock().unwrap();
             let chain_tip = chain.get_chain_tip()?;
-            let outpoints = graph.index.outpoints().iter().cloned();
+            let outpoints = graph.index.outpoints();
 
             match txout_cmd {
                 TxOutCmd::List {
@@ -556,7 +572,7 @@ where
                 } => {
                     let txouts = graph
                         .graph()
-                        .try_filter_chain_txouts(chain, chain_tip, outpoints)
+                        .try_filter_chain_txouts(chain, chain_tip, outpoints.iter().cloned())
                         .filter(|r| match r {
                             Ok((_, full_txo)) => match (spent, unspent) {
                                 (true, false) => full_txo.spent_by.is_some(),
@@ -613,7 +629,7 @@ where
                     // If we're unable to persist this, then we don't want to broadcast.
                     {
                         let db = &mut *db.lock().unwrap();
-                        db.stage_and_commit(C::from((
+                        db.append_changeset(&C::from((
                             local_chain::ChangeSet::default(),
                             indexed_tx_graph::ChangeSet::from(index_changeset),
                         )))?;
@@ -631,14 +647,14 @@ where
 
             match (broadcast)(chain_specific, &transaction) {
                 Ok(_) => {
-                    println!("Broadcasted Tx : {}", transaction.txid());
+                    println!("Broadcasted Tx : {}", transaction.compute_txid());
 
                     let keychain_changeset = graph.lock().unwrap().insert_tx(transaction);
 
                     // We know the tx is at least unconfirmed now. Note if persisting here fails,
                     // it's not a big deal since we can always find it again form
                     // blockchain.
-                    db.lock().unwrap().stage_and_commit(C::from((
+                    db.lock().unwrap().append_changeset(&C::from((
                         local_chain::ChangeSet::default(),
                         keychain_changeset,
                     )))?;
@@ -657,7 +673,10 @@ where
 }
 
 /// The initial state returned by [`init`].
-pub struct Init<CS: clap::Subcommand, S: clap::Args, C> {
+pub struct Init<CS: clap::Subcommand, S: clap::Args, C>
+where
+    C: Default + Merge + Serialize + DeserializeOwned + Debug + Send + Sync + 'static,
+{
     /// Arguments parsed by the cli.
     pub args: Args<CS, S>,
     /// Descriptor keymap.
@@ -665,7 +684,7 @@ pub struct Init<CS: clap::Subcommand, S: clap::Args, C> {
     /// Keychain-txout index.
     pub index: KeychainTxOutIndex<Keychain>,
     /// Persistence backend.
-    pub db: Mutex<Database<C>>,
+    pub db: Mutex<Store<C>>,
     /// Initial changeset.
     pub init_changeset: C,
 }
@@ -677,7 +696,14 @@ pub fn init<CS: clap::Subcommand, S: clap::Args, C>(
     db_default_path: &str,
 ) -> anyhow::Result<Init<CS, S, C>>
 where
-    C: Default + Append + Serialize + DeserializeOwned,
+    C: Default
+        + Merge
+        + Serialize
+        + DeserializeOwned
+        + Debug
+        + core::marker::Send
+        + core::marker::Sync
+        + 'static,
 {
     if std::env::var("BDK_DB_PATH").is_err() {
         std::env::set_var("BDK_DB_PATH", db_default_path);
@@ -687,9 +713,11 @@ where
 
     let mut index = KeychainTxOutIndex::<Keychain>::default();
 
+    // TODO: descriptors are already stored in the db, so we shouldn't re-insert
+    // them in the index here. However, the keymap is not stored in the database.
     let (descriptor, mut keymap) =
         Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, &args.descriptor)?;
-    index.add_keychain(Keychain::External, descriptor);
+    let _ = index.insert_descriptor(Keychain::External, descriptor)?;
 
     if let Some((internal_descriptor, internal_keymap)) = args
         .change_descriptor
@@ -698,7 +726,7 @@ where
         .transpose()?
     {
         keymap.extend(internal_keymap);
-        index.add_keychain(Keychain::Internal, internal_descriptor);
+        let _ = index.insert_descriptor(Keychain::Internal, internal_descriptor)?;
     }
 
     let mut db_backend = match Store::<C>::open_or_create_new(db_magic, &args.db_path) {
@@ -707,13 +735,13 @@ where
         Err(err) => return Err(anyhow::anyhow!("failed to init db backend: {:?}", err)),
     };
 
-    let init_changeset = db_backend.load_from_persistence()?.unwrap_or_default();
+    let init_changeset = db_backend.aggregate_changesets()?.unwrap_or_default();
 
     Ok(Init {
         args,
         keymap,
         index,
-        db: Mutex::new(Database::new(db_backend)),
+        db: Mutex::new(db_backend),
         init_changeset,
     })
 }

example-crates/example_electrum/src/main.rs

@@ -1,19 +1,20 @@
 use std::{
-    collections::BTreeMap,
     io::{self, Write},
     sync::Mutex,
 };
 
 use bdk_chain::{
-    bitcoin::{constants::genesis_block, Address, Network, OutPoint, Txid},
+    bitcoin::{constants::genesis_block, Address, Network, Txid},
+    collections::BTreeSet,
     indexed_tx_graph::{self, IndexedTxGraph},
-    keychain,
+    indexer::keychain_txout,
     local_chain::{self, LocalChain},
-    Append, ConfirmationHeightAnchor,
+    spk_client::{FullScanRequest, SyncRequest},
+    ConfirmationBlockTime, Merge,
 };
 use bdk_electrum::{
     electrum_client::{self, Client, ElectrumApi},
-    ElectrumExt, ElectrumUpdate,
+    BdkElectrumClient,
 };
 use example_cli::{
     anyhow::{self, Context},
@@ -99,7 +100,7 @@ pub struct ScanOptions {
 
 type ChangeSet = (
     local_chain::ChangeSet,
-    indexed_tx_graph::ChangeSet<ConfirmationHeightAnchor, keychain::ChangeSet<Keychain>>,
+    indexed_tx_graph::ChangeSet<ConfirmationBlockTime, keychain_txout::ChangeSet>,
 );
 
 fn main() -> anyhow::Result<()> {
@@ -145,44 +146,59 @@ fn main() -> anyhow::Result<()> {
         }
     };
 
-    let client = electrum_cmd.electrum_args().client(args.network)?;
+    let client = BdkElectrumClient::new(electrum_cmd.electrum_args().client(args.network)?);
 
-    let response = match electrum_cmd.clone() {
+    // Tell the electrum client about the txs we've already got locally so it doesn't re-download them
+    client.populate_tx_cache(&*graph.lock().unwrap());
+
+    let (chain_update, mut graph_update, keychain_update) = match electrum_cmd.clone() {
         ElectrumCommands::Scan {
             stop_gap,
             scan_options,
             ..
         } => {
-            let (keychain_spks, tip) = {
+            let request = {
                 let graph = &*graph.lock().unwrap();
                 let chain = &*chain.lock().unwrap();
 
-                let keychain_spks = graph
-                    .index
-                    .all_unbounded_spk_iters()
-                    .into_iter()
-                    .map(|(keychain, iter)| {
-                        let mut first = true;
-                        let spk_iter = iter.inspect(move |(i, _)| {
-                            if first {
-                                eprint!("\nscanning {}: ", keychain);
-                                first = false;
+                FullScanRequest::from_chain_tip(chain.tip())
+                    .set_spks_for_keychain(
+                        Keychain::External,
+                        graph
+                            .index
+                            .unbounded_spk_iter(Keychain::External)
+                            .into_iter()
+                            .flatten(),
+                    )
+                    .set_spks_for_keychain(
+                        Keychain::Internal,
+                        graph
+                            .index
+                            .unbounded_spk_iter(Keychain::Internal)
+                            .into_iter()
+                            .flatten(),
+                    )
+                    .inspect_spks_for_all_keychains({
+                        let mut once = BTreeSet::new();
+                        move |k, spk_i, _| {
+                            if once.insert(k) {
+                                eprint!("\nScanning {}: {} ", k, spk_i);
+                            } else {
+                                eprint!("{} ", spk_i);
                             }
-
-                            eprint!("{} ", i);
-                            let _ = io::stdout().flush();
-                        });
-                        (keychain, spk_iter)
+                            io::stdout().flush().expect("must flush");
+                        }
                     })
-                    .collect::<BTreeMap<_, _>>();
-
-                let tip = chain.tip();
-                (keychain_spks, tip)
             };
 
-            client
-                .full_scan(tip, keychain_spks, stop_gap, scan_options.batch_size)
-                .context("scanning the blockchain")?
+            let res = client
+                .full_scan::<_>(request, stop_gap, scan_options.batch_size, false)
+                .context("scanning the blockchain")?;
+            (
+                res.chain_update,
+                res.graph_update,
+                Some(res.last_active_indices),
+            )
         }
         ElectrumCommands::Sync {
             mut unused_spks,
@@ -195,7 +211,6 @@ fn main() -> anyhow::Result<()> {
             // Get a short lock on the tracker to get the spks we're interested in
             let graph = graph.lock().unwrap();
             let chain = chain.lock().unwrap();
-            let chain_tip = chain.tip().block_id();
 
             if !(all_spks || unused_spks || utxos || unconfirmed) {
                 unused_spks = true;
@@ -205,130 +220,135 @@ fn main() -> anyhow::Result<()> {
                 unused_spks = false;
             }
 
-            let mut spks: Box<dyn Iterator<Item = bdk_chain::bitcoin::ScriptBuf>> =
-                Box::new(core::iter::empty());
+            let chain_tip = chain.tip();
+            let mut request = SyncRequest::from_chain_tip(chain_tip.clone());
+
             if all_spks {
                 let all_spks = graph
                     .index
-                    .revealed_spks()
-                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                    .revealed_spks(..)
+                    .map(|(index, spk)| (index, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
-                    eprintln!("scanning {}:{}", k, i);
+                request = request.chain_spks(all_spks.into_iter().map(|((k, spk_i), spk)| {
+                    eprint!("Scanning {}: {}", k, spk_i);
                     spk
-                })));
+                }));
             }
             if unused_spks {
                 let unused_spks = graph
                     .index
                     .unused_spks()
-                    .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                    .map(|(index, spk)| (index, spk.to_owned()))
                     .collect::<Vec<_>>();
-                spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
-                    eprintln!(
-                        "Checking if address {} {}:{} has been used",
-                        Address::from_script(&spk, args.network).unwrap(),
-                        k,
-                        i,
-                    );
-                    spk
-                })));
+                request =
+                    request.chain_spks(unused_spks.into_iter().map(move |((k, spk_i), spk)| {
+                        eprint!(
+                            "Checking if address {} {}:{} has been used",
+                            Address::from_script(&spk, args.network).unwrap(),
+                            k,
+                            spk_i,
+                        );
+                        spk
+                    }));
             }
 
-            let mut outpoints: Box<dyn Iterator<Item = OutPoint>> = Box::new(core::iter::empty());
-
             if utxos {
-                let init_outpoints = graph.index.outpoints().iter().cloned();
+                let init_outpoints = graph.index.outpoints();
 
                 let utxos = graph
                     .graph()
-                    .filter_chain_unspents(&*chain, chain_tip, init_outpoints)
+                    .filter_chain_unspents(
+                        &*chain,
+                        chain_tip.block_id(),
+                        init_outpoints.iter().cloned(),
+                    )
                     .map(|(_, utxo)| utxo)
                     .collect::<Vec<_>>();
-
-                outpoints = Box::new(
-                    utxos
-                        .into_iter()
-                        .inspect(|utxo| {
-                            eprintln!(
-                                "Checking if outpoint {} (value: {}) has been spent",
-                                utxo.outpoint, utxo.txout.value
-                            );
-                        })
-                        .map(|utxo| utxo.outpoint),
-                );
+                request = request.chain_outpoints(utxos.into_iter().map(|utxo| {
+                    eprint!(
+                        "Checking if outpoint {} (value: {}) has been spent",
+                        utxo.outpoint, utxo.txout.value
+                    );
+                    utxo.outpoint
+                }));
             };
 
-            let mut txids: Box<dyn Iterator<Item = Txid>> = Box::new(core::iter::empty());
-
             if unconfirmed {
                 let unconfirmed_txids = graph
                     .graph()
-                    .list_chain_txs(&*chain, chain_tip)
+                    .list_canonical_txs(&*chain, chain_tip.block_id())
                     .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
                     .map(|canonical_tx| canonical_tx.tx_node.txid)
                     .collect::<Vec<Txid>>();
 
-                txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| {
-                    eprintln!("Checking if {} is confirmed yet", txid);
-                }));
+                request = request.chain_txids(
+                    unconfirmed_txids
+                        .into_iter()
+                        .inspect(|txid| eprint!("Checking if {} is confirmed yet", txid)),
+                );
             }
 
-            let tip = chain.tip();
+            let total_spks = request.spks.len();
+            let total_txids = request.txids.len();
+            let total_ops = request.outpoints.len();
+            request = request
+                .inspect_spks({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_spks as f32)
+                    }
+                })
+                .inspect_txids({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_txids as f32)
+                    }
+                })
+                .inspect_outpoints({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_ops as f32)
+                    }
+                });
+
+            let res = client
+                .sync(request, scan_options.batch_size, false)
+                .context("scanning the blockchain")?;
 
             // drop lock on graph and chain
             drop((graph, chain));
 
-            let electrum_update = client
-                .sync(tip, spks, txids, outpoints, scan_options.batch_size)
-                .context("scanning the blockchain")?;
-            (electrum_update, BTreeMap::new())
+            (res.chain_update, res.graph_update, None)
         }
     };
 
-    let (
-        ElectrumUpdate {
-            chain_update,
-            relevant_txids,
-        },
-        keychain_update,
-    ) = response;
-
-    let missing_txids = {
-        let graph = &*graph.lock().unwrap();
-        relevant_txids.missing_full_txs(graph.graph())
-    };
-
     let now = std::time::UNIX_EPOCH
         .elapsed()
         .expect("must get time")
         .as_secs();
-
-    let graph_update = relevant_txids.into_tx_graph(&client, Some(now), missing_txids)?;
+    let _ = graph_update.update_last_seen_unconfirmed(now);
 
     let db_changeset = {
         let mut chain = chain.lock().unwrap();
         let mut graph = graph.lock().unwrap();
 
-        let chain = chain.apply_update(chain_update)?;
+        let chain_changeset = chain.apply_update(chain_update)?;
 
-        let indexed_tx_graph = {
-            let mut changeset =
-                indexed_tx_graph::ChangeSet::<ConfirmationHeightAnchor, _>::default();
-            let (_, indexer) = graph.index.reveal_to_target_multi(&keychain_update);
-            changeset.append(indexed_tx_graph::ChangeSet {
-                indexer,
-                ..Default::default()
-            });
-            changeset.append(graph.apply_update(graph_update));
-            changeset
-        };
+        let mut indexed_tx_graph_changeset =
+            indexed_tx_graph::ChangeSet::<ConfirmationBlockTime, _>::default();
+        if let Some(keychain_update) = keychain_update {
+            let keychain_changeset = graph.index.reveal_to_target_multi(&keychain_update);
+            indexed_tx_graph_changeset.merge(keychain_changeset.into());
+        }
+        indexed_tx_graph_changeset.merge(graph.apply_update(graph_update));
 
-        (chain, indexed_tx_graph)
+        (chain_changeset, indexed_tx_graph_changeset)
     };
 
     let mut db = db.lock().unwrap();
-    db.stage(db_changeset);
-    db.commit()?;
+    db.append_changeset(&db_changeset)?;
     Ok(())
 }

example-crates/example_esplora/src/main.rs

@@ -1,15 +1,16 @@
 use std::{
-    collections::{BTreeMap, BTreeSet},
+    collections::BTreeSet,
     io::{self, Write},
     sync::Mutex,
 };
 
 use bdk_chain::{
-    bitcoin::{constants::genesis_block, Address, Network, OutPoint, ScriptBuf, Txid},
+    bitcoin::{constants::genesis_block, Address, Network, Txid},
     indexed_tx_graph::{self, IndexedTxGraph},
-    keychain,
+    indexer::keychain_txout,
     local_chain::{self, LocalChain},
-    Append, ConfirmationTimeHeightAnchor,
+    spk_client::{FullScanRequest, SyncRequest},
+    ConfirmationBlockTime, Merge,
 };
 
 use bdk_esplora::{esplora_client, EsploraExt};
@@ -21,11 +22,11 @@ use example_cli::{
 };
 
 const DB_MAGIC: &[u8] = b"bdk_example_esplora";
-const DB_PATH: &str = ".bdk_esplora_example.db";
+const DB_PATH: &str = "bdk_example_esplora.db";
 
 type ChangeSet = (
     local_chain::ChangeSet,
-    indexed_tx_graph::ChangeSet<ConfirmationTimeHeightAnchor, keychain::ChangeSet<Keychain>>,
+    indexed_tx_graph::ChangeSet<ConfirmationBlockTime, keychain_txout::ChangeSet>,
 );
 
 #[derive(Subcommand, Debug, Clone)]
@@ -60,6 +61,7 @@ enum EsploraCommands {
         esplora_args: EsploraArgs,
     },
 }
+
 impl EsploraCommands {
     fn esplora_args(&self) -> EsploraArgs {
         match self {
@@ -82,11 +84,11 @@ impl EsploraArgs {
             Network::Bitcoin => "https://blockstream.info/api",
             Network::Testnet => "https://blockstream.info/testnet/api",
             Network::Regtest => "http://localhost:3002",
-            Network::Signet => "https://mempool.space/signet/api",
+            Network::Signet => "http://signet.bitcoindevkit.net",
             _ => panic!("unsupported network"),
         });
 
-        let client = esplora_client::Builder::new(esplora_url).build_blocking()?;
+        let client = esplora_client::Builder::new(esplora_url).build_blocking();
         Ok(client)
     }
 }
@@ -94,7 +96,7 @@ impl EsploraArgs {
 #[derive(Parser, Debug, Clone, PartialEq)]
 pub struct ScanOptions {
     /// Max number of concurrent esplora server requests.
-    #[clap(long, default_value = "1")]
+    #[clap(long, default_value = "5")]
     pub parallel_requests: usize,
 }
 
@@ -149,59 +151,66 @@ fn main() -> anyhow::Result<()> {
     };
 
     let client = esplora_cmd.esplora_args().client(args.network)?;
-    // Prepare the `IndexedTxGraph` update based on whether we are scanning or syncing.
+    // Prepare the `IndexedTxGraph` and `LocalChain` updates based on whether we are scanning or
+    // syncing.
+    //
     // Scanning: We are iterating through spks of all keychains and scanning for transactions for
     //   each spk. We start with the lowest derivation index spk and stop scanning after `stop_gap`
     //   number of consecutive spks have no transaction history. A Scan is done in situations of
     //   wallet restoration. It is a special case. Applications should use "sync" style updates
     //   after an initial scan.
+    //
     // Syncing: We only check for specified spks, utxos and txids to update their confirmation
     //   status or fetch missing transactions.
-    let indexed_tx_graph_changeset = match &esplora_cmd {
+    let (local_chain_changeset, indexed_tx_graph_changeset) = match &esplora_cmd {
         EsploraCommands::Scan {
             stop_gap,
             scan_options,
             ..
         } => {
-            let keychain_spks = graph
-                .lock()
-                .expect("mutex must not be poisoned")
-                .index
-                .all_unbounded_spk_iters()
-                .into_iter()
-                // This `map` is purely for logging.
-                .map(|(keychain, iter)| {
-                    let mut first = true;
-                    let spk_iter = iter.inspect(move |(i, _)| {
-                        if first {
-                            eprint!("\nscanning {}: ", keychain);
-                            first = false;
+            let request = {
+                let chain_tip = chain.lock().expect("mutex must not be poisoned").tip();
+                let indexed_graph = &*graph.lock().expect("mutex must not be poisoned");
+                FullScanRequest::from_keychain_txout_index(chain_tip, &indexed_graph.index)
+                    .inspect_spks_for_all_keychains({
+                        let mut once = BTreeSet::<Keychain>::new();
+                        move |keychain, spk_i, _| {
+                            if once.insert(keychain) {
+                                eprint!("\nscanning {}: ", keychain);
+                            }
+                            eprint!("{} ", spk_i);
+                            // Flush early to ensure we print at every iteration.
+                            let _ = io::stderr().flush();
                         }
-                        eprint!("{} ", i);
-                        // Flush early to ensure we print at every iteration.
-                        let _ = io::stderr().flush();
-                    });
-                    (keychain, spk_iter)
-                })
-                .collect::<BTreeMap<_, _>>();
+                    })
+            };
 
             // The client scans keychain spks for transaction histories, stopping after `stop_gap`
             // is reached. It returns a `TxGraph` update (`graph_update`) and a structure that
             // represents the last active spk derivation indices of keychains
             // (`keychain_indices_update`).
-            let (graph_update, last_active_indices) = client
-                .full_scan(keychain_spks, *stop_gap, scan_options.parallel_requests)
+            let mut update = client
+                .full_scan(request, *stop_gap, scan_options.parallel_requests)
                 .context("scanning for transactions")?;
 
+            // We want to keep track of the latest time a transaction was seen unconfirmed.
+            let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+            let _ = update.graph_update.update_last_seen_unconfirmed(now);
+
             let mut graph = graph.lock().expect("mutex must not be poisoned");
+            let mut chain = chain.lock().expect("mutex must not be poisoned");
             // Because we did a stop gap based scan we are likely to have some updates to our
             // deriviation indices. Usually before a scan you are on a fresh wallet with no
             // addresses derived so we need to derive up to last active addresses the scan found
             // before adding the transactions.
-            let (_, index_changeset) = graph.index.reveal_to_target_multi(&last_active_indices);
-            let mut indexed_tx_graph_changeset = graph.apply_update(graph_update);
-            indexed_tx_graph_changeset.append(index_changeset.into());
-            indexed_tx_graph_changeset
+            (chain.apply_update(update.chain_update)?, {
+                let index_changeset = graph
+                    .index
+                    .reveal_to_target_multi(&update.last_active_indices);
+                let mut indexed_tx_graph_changeset = graph.apply_update(update.graph_update);
+                indexed_tx_graph_changeset.merge(index_changeset.into());
+                indexed_tx_graph_changeset
+            })
         }
         EsploraCommands::Sync {
             mut unused_spks,
@@ -222,64 +231,67 @@ fn main() -> anyhow::Result<()> {
                 unused_spks = false;
             }
 
+            let local_tip = chain.lock().expect("mutex must not be poisoned").tip();
             // Spks, outpoints and txids we want updates on will be accumulated here.
-            let mut spks: Box<dyn Iterator<Item = ScriptBuf>> = Box::new(core::iter::empty());
-            let mut outpoints: Box<dyn Iterator<Item = OutPoint>> = Box::new(core::iter::empty());
-            let mut txids: Box<dyn Iterator<Item = Txid>> = Box::new(core::iter::empty());
+            let mut request = SyncRequest::from_chain_tip(local_tip.clone());
 
             // Get a short lock on the structures to get spks, utxos, and txs that we are interested
             // in.
             {
                 let graph = graph.lock().unwrap();
                 let chain = chain.lock().unwrap();
-                let chain_tip = chain.tip().block_id();
 
                 if *all_spks {
                     let all_spks = graph
                         .index
-                        .revealed_spks()
-                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                        .revealed_spks(..)
+                        .map(|((k, i), spk)| (k, i, spk.to_owned()))
                         .collect::<Vec<_>>();
-                    spks = Box::new(spks.chain(all_spks.into_iter().map(|(k, i, spk)| {
-                        eprintln!("scanning {}:{}", k, i);
+                    request = request.chain_spks(all_spks.into_iter().map(|(k, i, spk)| {
+                        eprint!("scanning {}:{}", k, i);
                         // Flush early to ensure we print at every iteration.
                         let _ = io::stderr().flush();
                         spk
-                    })));
+                    }));
                 }
                 if unused_spks {
                     let unused_spks = graph
                         .index
                         .unused_spks()
-                        .map(|(k, i, spk)| (k, i, spk.to_owned()))
+                        .map(|(index, spk)| (index, spk.to_owned()))
                         .collect::<Vec<_>>();
-                    spks = Box::new(spks.chain(unused_spks.into_iter().map(|(k, i, spk)| {
-                        eprintln!(
-                            "Checking if address {} {}:{} has been used",
-                            Address::from_script(&spk, args.network).unwrap(),
-                            k,
-                            i,
-                        );
-                        // Flush early to ensure we print at every iteration.
-                        let _ = io::stderr().flush();
-                        spk
-                    })));
+                    request =
+                        request.chain_spks(unused_spks.into_iter().map(move |((k, i), spk)| {
+                            eprint!(
+                                "Checking if address {} {}:{} has been used",
+                                Address::from_script(&spk, args.network).unwrap(),
+                                k,
+                                i,
+                            );
+                            // Flush early to ensure we print at every iteration.
+                            let _ = io::stderr().flush();
+                            spk
+                        }));
                 }
                 if utxos {
                     // We want to search for whether the UTXO is spent, and spent by which
                     // transaction. We provide the outpoint of the UTXO to
                     // `EsploraExt::update_tx_graph_without_keychain`.
-                    let init_outpoints = graph.index.outpoints().iter().cloned();
+                    let init_outpoints = graph.index.outpoints();
                     let utxos = graph
                         .graph()
-                        .filter_chain_unspents(&*chain, chain_tip, init_outpoints)
+                        .filter_chain_unspents(
+                            &*chain,
+                            local_tip.block_id(),
+                            init_outpoints.iter().cloned(),
+                        )
                         .map(|(_, utxo)| utxo)
                         .collect::<Vec<_>>();
-                    outpoints = Box::new(
+                    request = request.chain_outpoints(
                         utxos
                             .into_iter()
                             .inspect(|utxo| {
-                                eprintln!(
+                                eprint!(
                                     "Checking if outpoint {} (value: {}) has been spent",
                                     utxo.outpoint, utxo.txout.value
                                 );
@@ -295,56 +307,60 @@ fn main() -> anyhow::Result<()> {
                     // `EsploraExt::update_tx_graph_without_keychain`.
                     let unconfirmed_txids = graph
                         .graph()
-                        .list_chain_txs(&*chain, chain_tip)
+                        .list_canonical_txs(&*chain, local_tip.block_id())
                         .filter(|canonical_tx| !canonical_tx.chain_position.is_confirmed())
                         .map(|canonical_tx| canonical_tx.tx_node.txid)
                         .collect::<Vec<Txid>>();
-                    txids = Box::new(unconfirmed_txids.into_iter().inspect(|txid| {
-                        eprintln!("Checking if {} is confirmed yet", txid);
+                    request = request.chain_txids(unconfirmed_txids.into_iter().inspect(|txid| {
+                        eprint!("Checking if {} is confirmed yet", txid);
                         // Flush early to ensure we print at every iteration.
                         let _ = io::stderr().flush();
                     }));
                 }
             }
 
-            let graph_update =
-                client.sync(spks, txids, outpoints, scan_options.parallel_requests)?;
+            let total_spks = request.spks.len();
+            let total_txids = request.txids.len();
+            let total_ops = request.outpoints.len();
+            request = request
+                .inspect_spks({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_spks as f32)
+                    }
+                })
+                .inspect_txids({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_txids as f32)
+                    }
+                })
+                .inspect_outpoints({
+                    let mut visited = 0;
+                    move |_| {
+                        visited += 1;
+                        eprintln!(" [ {:>6.2}% ]", (visited * 100) as f32 / total_ops as f32)
+                    }
+                });
+            let mut update = client.sync(request, scan_options.parallel_requests)?;
 
-            graph.lock().unwrap().apply_update(graph_update)
+            // Update last seen unconfirmed
+            let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+            let _ = update.graph_update.update_last_seen_unconfirmed(now);
+
+            (
+                chain.lock().unwrap().apply_update(update.chain_update)?,
+                graph.lock().unwrap().apply_update(update.graph_update),
+            )
         }
     };
 
     println!();
 
-    // Now that we're done updating the `IndexedTxGraph`, it's time to update the `LocalChain`! We
-    // want the `LocalChain` to have data about all the anchors in the `TxGraph` - for this reason,
-    // we want retrieve the blocks at the heights of the newly added anchors that are missing from
-    // our view of the chain.
-    let (missing_block_heights, tip) = {
-        let chain = &*chain.lock().unwrap();
-        let missing_block_heights = indexed_tx_graph_changeset
-            .graph
-            .missing_heights_from(chain)
-            .collect::<BTreeSet<_>>();
-        let tip = chain.tip();
-        (missing_block_heights, tip)
-    };
-
-    println!("prev tip: {}", tip.height());
-    println!("missing block heights: {:?}", missing_block_heights);
-
-    // Here, we actually fetch the missing blocks and create a `local_chain::Update`.
-    let chain_changeset = {
-        let chain_update = client
-            .update_local_chain(tip, missing_block_heights)
-            .context("scanning for blocks")?;
-        println!("new tip: {}", chain_update.tip.height());
-        chain.lock().unwrap().apply_update(chain_update)?
-    };
-
     // We persist the changes
     let mut db = db.lock().unwrap();
-    db.stage((chain_changeset, indexed_tx_graph_changeset));
-    db.commit()?;
+    db.append_changeset(&(local_chain_changeset, indexed_tx_graph_changeset))?;
     Ok(())
 }

example-crates/wallet_electrum/Cargo.toml

@@ -4,7 +4,6 @@ version = "0.2.0"
 edition = "2021"
 
 [dependencies]
-bdk = { path = "../../crates/bdk" }
+bdk_wallet = { path = "../../crates/wallet", features = ["file_store"] }
 bdk_electrum = { path = "../../crates/electrum" }
-bdk_file_store = { path = "../../crates/file_store" }
 anyhow = "1"

example-crates/wallet_electrum/src/main.rs

@@ -1,82 +1,80 @@
-const DB_MAGIC: &str = "bdk_wallet_electrum_example";
-const SEND_AMOUNT: u64 = 5000;
-const STOP_GAP: usize = 50;
-const BATCH_SIZE: usize = 5;
-
+use bdk_wallet::file_store::Store;
+use bdk_wallet::Wallet;
 use std::io::Write;
 use std::str::FromStr;
 
-use bdk::bitcoin::Address;
-use bdk::wallet::Update;
-use bdk::SignOptions;
-use bdk::{bitcoin::Network, Wallet};
-use bdk_electrum::{
-    electrum_client::{self, ElectrumApi},
-    ElectrumExt, ElectrumUpdate,
-};
-use bdk_file_store::Store;
+use bdk_electrum::electrum_client;
+use bdk_electrum::BdkElectrumClient;
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::bitcoin::{Address, Amount};
+use bdk_wallet::chain::collections::HashSet;
+use bdk_wallet::{KeychainKind, SignOptions};
+
+const DB_MAGIC: &str = "bdk_wallet_electrum_example";
+const SEND_AMOUNT: Amount = Amount::from_sat(5000);
+const STOP_GAP: usize = 50;
+const BATCH_SIZE: usize = 5;
+
+const NETWORK: Network = Network::Testnet;
+const EXTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+const INTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+const ELECTRUM_URL: &str = "ssl://electrum.blockstream.info:60002";
 
 fn main() -> Result<(), anyhow::Error> {
-    let db_path = std::env::temp_dir().join("bdk-electrum-example");
-    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+    let db_path = "bdk-electrum-example.db";
 
-    let mut wallet = Wallet::new_or_load(
-        external_descriptor,
-        Some(internal_descriptor),
-        db,
-        Network::Testnet,
-    )?;
+    let mut db = Store::<bdk_wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
 
-    let address = wallet.try_get_address(bdk::wallet::AddressIndex::New)?;
+    let wallet_opt = Wallet::load()
+        .descriptors(EXTERNAL_DESC, INTERNAL_DESC)
+        .network(NETWORK)
+        .load_wallet(&mut db)?;
+    let mut wallet = match wallet_opt {
+        Some(wallet) => wallet,
+        None => Wallet::create(EXTERNAL_DESC, INTERNAL_DESC)
+            .network(NETWORK)
+            .create_wallet(&mut db)?,
+    };
+
+    let address = wallet.next_unused_address(KeychainKind::External);
+    wallet.persist(&mut db)?;
     println!("Generated Address: {}", address);
 
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance before syncing: {} sats", balance.total());
 
     print!("Syncing...");
-    let client = electrum_client::Client::new("ssl://electrum.blockstream.info:60002")?;
+    let client = BdkElectrumClient::new(electrum_client::Client::new(ELECTRUM_URL)?);
 
-    let prev_tip = wallet.latest_checkpoint();
-    let keychain_spks = wallet
-        .all_unbounded_spk_iters()
-        .into_iter()
-        .map(|(k, k_spks)| {
-            let mut once = Some(());
-            let mut stdout = std::io::stdout();
-            let k_spks = k_spks
-                .inspect(move |(spk_i, _)| match once.take() {
-                    Some(_) => print!("\nScanning keychain [{:?}]", k),
-                    None => print!(" {:<3}", spk_i),
-                })
-                .inspect(move |_| stdout.flush().expect("must flush"));
-            (k, k_spks)
+    // Populate the electrum client's transaction cache so it doesn't redownload transaction we
+    // already have.
+    client.populate_tx_cache(wallet.tx_graph());
+
+    let request = wallet
+        .start_full_scan()
+        .inspect_spks_for_all_keychains({
+            let mut once = HashSet::<KeychainKind>::new();
+            move |k, spk_i, _| {
+                if once.insert(k) {
+                    print!("\nScanning keychain [{:?}]", k)
+                } else {
+                    print!(" {:<3}", spk_i)
+                }
+            }
         })
-        .collect();
+        .inspect_spks_for_all_keychains(|_, _, _| std::io::stdout().flush().expect("must flush"));
 
-    let (
-        ElectrumUpdate {
-            chain_update,
-            relevant_txids,
-        },
-        keychain_update,
-    ) = client.full_scan(prev_tip, keychain_spks, STOP_GAP, BATCH_SIZE)?;
+    let mut update = client.full_scan(request, STOP_GAP, BATCH_SIZE, false)?;
+
+    let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+    let _ = update.graph_update.update_last_seen_unconfirmed(now);
 
     println!();
 
-    let missing = relevant_txids.missing_full_txs(wallet.as_ref());
-    let graph_update = relevant_txids.into_confirmation_time_tx_graph(&client, None, missing)?;
+    wallet.apply_update(update)?;
+    wallet.persist(&mut db)?;
 
-    let wallet_update = Update {
-        last_active_indices: keychain_update,
-        graph: graph_update,
-        chain: Some(chain_update),
-    };
-    wallet.apply_update(wallet_update)?;
-    wallet.commit()?;
-
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance after syncing: {} sats", balance.total());
 
     if balance.total() < SEND_AMOUNT {
@@ -99,9 +97,9 @@ fn main() -> Result<(), anyhow::Error> {
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);
 
-    let tx = psbt.extract_tx();
+    let tx = psbt.extract_tx()?;
     client.transaction_broadcast(&tx)?;
-    println!("Tx broadcasted! Txid: {}", tx.txid());
+    println!("Tx broadcasted! Txid: {}", tx.compute_txid());
 
     Ok(())
 }

example-crates/wallet_esplora_async/Cargo.toml

@@ -6,8 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk = { path = "../../crates/bdk" }
+bdk_wallet = { path = "../../crates/wallet", features = ["rusqlite"] }
 bdk_esplora = { path = "../../crates/esplora", features = ["async-https"] }
-bdk_file_store = { path = "../../crates/file_store" }
 tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
 anyhow = "1"

example-crates/wallet_esplora_async/src/main.rs

@@ -1,73 +1,70 @@
-use std::{io::Write, str::FromStr};
+use std::{collections::BTreeSet, io::Write};
 
-use bdk::{
-    bitcoin::{Address, Network},
-    wallet::{AddressIndex, Update},
-    SignOptions, Wallet,
-};
+use anyhow::Ok;
 use bdk_esplora::{esplora_client, EsploraAsyncExt};
-use bdk_file_store::Store;
+use bdk_wallet::{
+    bitcoin::{Amount, Network},
+    rusqlite::Connection,
+    KeychainKind, SignOptions, Wallet,
+};
 
-const DB_MAGIC: &str = "bdk_wallet_esplora_async_example";
-const SEND_AMOUNT: u64 = 5000;
-const STOP_GAP: usize = 50;
+const SEND_AMOUNT: Amount = Amount::from_sat(5000);
+const STOP_GAP: usize = 5;
 const PARALLEL_REQUESTS: usize = 5;
 
+const DB_PATH: &str = "bdk-example-esplora-async.sqlite";
+const NETWORK: Network = Network::Signet;
+const EXTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+const INTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+const ESPLORA_URL: &str = "http://signet.bitcoindevkit.net";
+
 #[tokio::main]
 async fn main() -> Result<(), anyhow::Error> {
-    let db_path = std::env::temp_dir().join("bdk-esplora-async-example");
-    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+    let mut conn = Connection::open(DB_PATH)?;
 
-    let mut wallet = Wallet::new_or_load(
-        external_descriptor,
-        Some(internal_descriptor),
-        db,
-        Network::Testnet,
-    )?;
+    let wallet_opt = Wallet::load()
+        .descriptors(EXTERNAL_DESC, INTERNAL_DESC)
+        .network(NETWORK)
+        .load_wallet(&mut conn)?;
+    let mut wallet = match wallet_opt {
+        Some(wallet) => wallet,
+        None => Wallet::create(EXTERNAL_DESC, INTERNAL_DESC)
+            .network(NETWORK)
+            .create_wallet(&mut conn)?,
+    };
 
-    let address = wallet.try_get_address(AddressIndex::New)?;
-    println!("Generated Address: {}", address);
+    let address = wallet.next_unused_address(KeychainKind::External);
+    wallet.persist(&mut conn)?;
+    println!("Next unused address: ({}) {}", address.index, address);
 
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance before syncing: {} sats", balance.total());
 
     print!("Syncing...");
-    let client =
-        esplora_client::Builder::new("https://blockstream.info/testnet/api").build_async()?;
+    let client = esplora_client::Builder::new(ESPLORA_URL).build_async()?;
 
-    let prev_tip = wallet.latest_checkpoint();
-    let keychain_spks = wallet
-        .all_unbounded_spk_iters()
-        .into_iter()
-        .map(|(k, k_spks)| {
-            let mut once = Some(());
-            let mut stdout = std::io::stdout();
-            let k_spks = k_spks
-                .inspect(move |(spk_i, _)| match once.take() {
-                    Some(_) => print!("\nScanning keychain [{:?}]", k),
-                    None => print!(" {:<3}", spk_i),
-                })
-                .inspect(move |_| stdout.flush().expect("must flush"));
-            (k, k_spks)
-        })
-        .collect();
-    let (update_graph, last_active_indices) = client
-        .full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)
+    let request = wallet.start_full_scan().inspect_spks_for_all_keychains({
+        let mut once = BTreeSet::<KeychainKind>::new();
+        move |keychain, spk_i, _| {
+            if once.insert(keychain) {
+                print!("\nScanning keychain [{:?}] ", keychain);
+            }
+            print!(" {:<3}", spk_i);
+            std::io::stdout().flush().expect("must flush")
+        }
+    });
+
+    let mut update = client
+        .full_scan(request, STOP_GAP, PARALLEL_REQUESTS)
         .await?;
-    let missing_heights = update_graph.missing_heights(wallet.local_chain());
-    let chain_update = client.update_local_chain(prev_tip, missing_heights).await?;
-    let update = Update {
-        last_active_indices,
-        graph: update_graph,
-        chain: Some(chain_update),
-    };
+    let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+    let _ = update.graph_update.update_last_seen_unconfirmed(now);
+
     wallet.apply_update(update)?;
-    wallet.commit()?;
+    wallet.persist(&mut conn)?;
     println!();
 
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance after syncing: {} sats", balance.total());
 
     if balance.total() < SEND_AMOUNT {
@@ -78,21 +75,18 @@ async fn main() -> Result<(), anyhow::Error> {
         std::process::exit(0);
     }
 
-    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?
-        .require_network(Network::Testnet)?;
-
     let mut tx_builder = wallet.build_tx();
     tx_builder
-        .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
+        .add_recipient(address.script_pubkey(), SEND_AMOUNT)
         .enable_rbf();
 
     let mut psbt = tx_builder.finish()?;
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);
 
-    let tx = psbt.extract_tx();
+    let tx = psbt.extract_tx()?;
     client.broadcast(&tx).await?;
-    println!("Tx broadcasted! Txid: {}", tx.txid());
+    println!("Tx broadcasted! Txid: {}", tx.compute_txid());
 
     Ok(())
 }

example-crates/wallet_esplora_blocking/Cargo.toml

@@ -7,7 +7,6 @@ publish = false
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk = { path = "../../crates/bdk" }
+bdk_wallet = { path = "../../crates/wallet", features = ["file_store"] }
 bdk_esplora = { path = "../../crates/esplora", features = ["blocking"] }
-bdk_file_store = { path = "../../crates/file_store" }
 anyhow = "1"

example-crates/wallet_esplora_blocking/src/main.rs

@@ -1,73 +1,72 @@
-const DB_MAGIC: &str = "bdk_wallet_esplora_example";
-const SEND_AMOUNT: u64 = 1000;
-const STOP_GAP: usize = 5;
-const PARALLEL_REQUESTS: usize = 1;
+use std::{collections::BTreeSet, io::Write};
 
-use std::{io::Write, str::FromStr};
-
-use bdk::{
-    bitcoin::{Address, Network},
-    wallet::{AddressIndex, Update},
-    SignOptions, Wallet,
-};
 use bdk_esplora::{esplora_client, EsploraExt};
-use bdk_file_store::Store;
+use bdk_wallet::{
+    bitcoin::{Amount, Network},
+    file_store::Store,
+    KeychainKind, SignOptions, Wallet,
+};
+
+const DB_MAGIC: &str = "bdk_wallet_esplora_example";
+const DB_PATH: &str = "bdk-example-esplora-blocking.db";
+const SEND_AMOUNT: Amount = Amount::from_sat(5000);
+const STOP_GAP: usize = 5;
+const PARALLEL_REQUESTS: usize = 5;
+
+const NETWORK: Network = Network::Signet;
+const EXTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
+const INTERNAL_DESC: &str = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+const ESPLORA_URL: &str = "http://signet.bitcoindevkit.net";
 
 fn main() -> Result<(), anyhow::Error> {
-    let db_path = std::env::temp_dir().join("bdk-esplora-example");
-    let db = Store::<bdk::wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), db_path)?;
-    let external_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/0/*)";
-    let internal_descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/1'/0'/1/*)";
+    let mut db = Store::<bdk_wallet::ChangeSet>::open_or_create_new(DB_MAGIC.as_bytes(), DB_PATH)?;
 
-    let mut wallet = Wallet::new_or_load(
-        external_descriptor,
-        Some(internal_descriptor),
-        db,
-        Network::Testnet,
-    )?;
+    let wallet_opt = Wallet::load()
+        .descriptors(EXTERNAL_DESC, INTERNAL_DESC)
+        .network(NETWORK)
+        .load_wallet(&mut db)?;
+    let mut wallet = match wallet_opt {
+        Some(wallet) => wallet,
+        None => Wallet::create(EXTERNAL_DESC, INTERNAL_DESC)
+            .network(NETWORK)
+            .create_wallet(&mut db)?,
+    };
 
-    let address = wallet.try_get_address(AddressIndex::New)?;
-    println!("Generated Address: {}", address);
+    let address = wallet.next_unused_address(KeychainKind::External);
+    wallet.persist(&mut db)?;
+    println!(
+        "Next unused address: ({}) {}",
+        address.index, address.address
+    );
 
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance before syncing: {} sats", balance.total());
 
     print!("Syncing...");
-    let client =
-        esplora_client::Builder::new("https://blockstream.info/testnet/api").build_blocking()?;
+    let client = esplora_client::Builder::new(ESPLORA_URL).build_blocking();
 
-    let prev_tip = wallet.latest_checkpoint();
-    let keychain_spks = wallet
-        .all_unbounded_spk_iters()
-        .into_iter()
-        .map(|(k, k_spks)| {
-            let mut once = Some(());
-            let mut stdout = std::io::stdout();
-            let k_spks = k_spks
-                .inspect(move |(spk_i, _)| match once.take() {
-                    Some(_) => print!("\nScanning keychain [{:?}]", k),
-                    None => print!(" {:<3}", spk_i),
-                })
-                .inspect(move |_| stdout.flush().expect("must flush"));
-            (k, k_spks)
-        })
-        .collect();
+    let request = wallet.start_full_scan().inspect_spks_for_all_keychains({
+        let mut once = BTreeSet::<KeychainKind>::new();
+        move |keychain, spk_i, _| {
+            if once.insert(keychain) {
+                print!("\nScanning keychain [{:?}] ", keychain);
+            }
+            print!(" {:<3}", spk_i);
+            std::io::stdout().flush().expect("must flush")
+        }
+    });
 
-    let (update_graph, last_active_indices) =
-        client.full_scan(keychain_spks, STOP_GAP, PARALLEL_REQUESTS)?;
-    let missing_heights = update_graph.missing_heights(wallet.local_chain());
-    let chain_update = client.update_local_chain(prev_tip, missing_heights)?;
-    let update = Update {
-        last_active_indices,
-        graph: update_graph,
-        chain: Some(chain_update),
-    };
+    let mut update = client.full_scan(request, STOP_GAP, PARALLEL_REQUESTS)?;
+    let now = std::time::UNIX_EPOCH.elapsed().unwrap().as_secs();
+    let _ = update.graph_update.update_last_seen_unconfirmed(now);
 
     wallet.apply_update(update)?;
-    wallet.commit()?;
+    if let Some(changeset) = wallet.take_staged() {
+        db.append_changeset(&changeset)?;
+    }
     println!();
 
-    let balance = wallet.get_balance();
+    let balance = wallet.balance();
     println!("Wallet balance after syncing: {} sats", balance.total());
 
     if balance.total() < SEND_AMOUNT {
@@ -78,21 +77,18 @@ fn main() -> Result<(), anyhow::Error> {
         std::process::exit(0);
     }
 
-    let faucet_address = Address::from_str("mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt")?
-        .require_network(Network::Testnet)?;
-
     let mut tx_builder = wallet.build_tx();
     tx_builder
-        .add_recipient(faucet_address.script_pubkey(), SEND_AMOUNT)
+        .add_recipient(address.script_pubkey(), SEND_AMOUNT)
         .enable_rbf();
 
     let mut psbt = tx_builder.finish()?;
     let finalized = wallet.sign(&mut psbt, SignOptions::default())?;
     assert!(finalized);
 
-    let tx = psbt.extract_tx();
+    let tx = psbt.extract_tx()?;
     client.broadcast(&tx)?;
-    println!("Tx broadcasted! Txid: {}", tx.txid());
+    println!("Tx broadcasted! Txid: {}", tx.compute_txid());
 
     Ok(())
 }

example-crates/wallet_rpc/Cargo.toml

@@ -6,8 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk = { path = "../../crates/bdk" }
-bdk_file_store = { path = "../../crates/file_store" }
+bdk_wallet = { path = "../../crates/wallet", features = ["file_store"] }
 bdk_bitcoind_rpc = { path = "../../crates/bitcoind_rpc" }
 
 anyhow = "1"

example-crates/wallet_rpc/README.md

@@ -4,7 +4,7 @@
 $ cargo run --bin wallet_rpc -- --help
 
 wallet_rpc 0.1.0
-Bitcoind RPC example using `bdk::Wallet`
+Bitcoind RPC example using `bdk_wallet::Wallet`
 
 USAGE:
     wallet_rpc [OPTIONS] <DESCRIPTOR> [CHANGE_DESCRIPTOR]