Merge bitcoindevkit/bdk#1472: Bump bdk version to 1.0.0-alpha.13

e21affdbbb Bump bdk version to 1.0.0-alpha.13 (Steve Myers)

Pull request description:

  ### Description

  Bump bdk version to 1.0.0-alpha.13

  bdk_chain to 0.16.0
  bdk_bitcoind_rpc to 0.12.0
  bdk_electrum to 0.15.0
  bdk_esplora to 0.15.0
  bdk_file_store to 0.13.0
  bdk_sqlite keep at 0.2.0
  bdk_testenv to 0.6.0
  bdk_hwi to 0.3.0

  fixes #1471

Top commit has no ACKs.

Tree-SHA512: 987adf5084dc84261fb3767d8b1f8ebe3dd94a8a4eb30b8529b70c055e4f122cb48063d205e90831169c2d7b2a1509aef1966857bd6b67e78cfb0d52144822d9

.github/dependabot.yml

@@ -0,0 +1,8 @@
+# Set update schedule for GitHub Actions
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every week
+      interval: "weekly"

.github/workflows/code_coverage.yml

@@ -19,7 +19,7 @@ jobs:
       - name: Install Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
-            toolchain: "1.65.0"
+            toolchain: stable
             override: true
             profile: minimal
             components: llvm-tools-preview
@@ -27,6 +27,7 @@ jobs:
         uses: Swatinem/rust-cache@v2.2.1
       - name: Install grcov
         run: if [[ ! -e ~/.cargo/bin/grcov ]]; then cargo install grcov; fi
+        # TODO: re-enable the hwi tests
       - name: Build simulator image
         run: docker build -t hwi/ledger_emulator ./ci -f ci/Dockerfile.ledger
       - name: Run simulator image

.github/workflows/cont_integration.yml

@@ -12,7 +12,7 @@ jobs:
         rust:
           - version: stable
             clippy: true
-          - version: 1.57.0 # MSRV
+          - version: 1.63.0 # MSRV
         features:
           - --no-default-features
           - --all-features
@@ -27,6 +27,14 @@ jobs:
             profile: minimal
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.2.1
+      - name: Pin dependencies for MSRV
+        if: matrix.rust.version == '1.63.0'
+        run: |
+          cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
+          cargo update -p time --precise "0.3.20"
+          cargo update -p home --precise "0.5.5"
+          cargo update -p proptest --precise "1.2.0"
+          cargo update -p url --precise "2.5.0"
       - name: Build
         run: cargo build ${{ matrix.features }}
       - name: Test
@@ -50,15 +58,15 @@ jobs:
       - name: Check bdk_chain
         working-directory: ./crates/chain
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,hashbrown
-      - name: Check bdk
-        working-directory: ./crates/bdk
+        run: cargo check --no-default-features --features miniscript/no-std,hashbrown
+      - name: Check bdk wallet
+        working-directory: ./crates/wallet
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown
+        run: cargo check --no-default-features --features miniscript/no-std,bdk_chain/hashbrown
       - name: Check esplora
         working-directory: ./crates/esplora
         # TODO "--target thumbv6m-none-eabi" should work but currently does not
-        run: cargo check --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown
+        run: cargo check --no-default-features --features miniscript/no-std,bdk_chain/hashbrown
 
   check-wasm:
     name: Check WASM
@@ -71,7 +79,6 @@ jobs:
         uses: actions/checkout@v2
         # Install a recent version of clang that supports wasm32
       - run: wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - || exit 1
-      - run: sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal-10 main" || exit 1
       - run: sudo apt-get update || exit 1
       - run: sudo apt-get install -y libclang-common-10-dev clang-10 libc6-dev-i386 || exit 1
       - name: Install Rust toolchain
@@ -83,12 +90,12 @@ jobs:
             target: "wasm32-unknown-unknown"
       - name: Rust Cache
         uses: Swatinem/rust-cache@v2.2.1
-      - name: Check bdk
-        working-directory: ./crates/bdk
-        run: cargo check --target wasm32-unknown-unknown --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
+      - name: Check bdk wallet
+        working-directory: ./crates/wallet
+        run: cargo check --target wasm32-unknown-unknown --no-default-features --features miniscript/no-std,bdk_chain/hashbrown,dev-getrandom-wasm
       - name: Check esplora
         working-directory: ./crates/esplora
-        run: cargo check --target wasm32-unknown-unknown --no-default-features --features bitcoin/no-std,miniscript/no-std,bdk_chain/hashbrown,async
+        run: cargo check --target wasm32-unknown-unknown --no-default-features --features miniscript/no-std,bdk_chain/hashbrown,async
 
   fmt:
     name: Rust fmt
@@ -112,9 +119,7 @@ jobs:
       - uses: actions/checkout@v1
       - uses: actions-rs/toolchain@v1
         with:
-            # we pin clippy instead of using "stable" so that our CI doesn't break
-            # at each new cargo release
-            toolchain: "1.67.0"
+            toolchain: 1.78.0
             components: clippy
             override: true
       - name: Rust Cache

.github/workflows/nightly_docs.yml

@@ -10,7 +10,7 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v2
       - name: Set default toolchain
-        run: rustup default nightly-2022-12-14
+        run: rustup default nightly-2024-05-12
       - name: Set profile
         run: rustup set profile minimal
       - name: Update toolchain

.gitignore

@@ -4,3 +4,7 @@ Cargo.lock
 
 *.swp
 .idea
+
+# Example persisted files.
+*.db
+*.sqlite*

CHANGELOG.md

@@ -158,7 +158,7 @@ BDK and LDK together.
 - Add the ability to specify which leaves to sign in a taproot transaction through `TapLeavesOptions` in `SignOptions`
 - Add the ability to specify whether a taproot transaction should be signed using the internal key or not, using `sign_with_tap_internal_key` in `SignOptions`
 - Consolidate params `fee_amount` and `amount_needed` in `target_amount` in `CoinSelectionAlgorithm::coin_select` signature.
-- Change the meaning of the `fee_amount` field inside `CoinSelectionResult`: from now on the `fee_amount` will represent only the fees asociated with the utxos in the `selected` field of `CoinSelectionResult`.
+- Change the meaning of the `fee_amount` field inside `CoinSelectionResult`: from now on the `fee_amount` will represent only the fees associated with the utxos in the `selected` field of `CoinSelectionResult`.
 - New `RpcBlockchain` implementation with various fixes.
 - Return balance in separate categories, namely `confirmed`, `trusted_pending`, `untrusted_pending` & `immature`.
 
@@ -449,7 +449,7 @@ final transaction is created by calling `finish` on the builder.
 #### Changed
 - Simplify the architecture of blockchain traits
 - Improve sync
-- Remove unused varaint HeaderParseFail
+- Remove unused variant `HeaderParseFail`
 
 ### CLI
 #### Added
@@ -517,7 +517,7 @@ final transaction is created by calling `finish` on the builder.
 - Default to SIGHASH_ALL if not specified
 - Replace ChangeSpendPolicy::filter_utxos with a predicate
 - Make 'unspendable' into a HashSet
-- Stop implicitly enforcing manaul selection by .add_utxo
+- Stop implicitly enforcing manual selection by .add_utxo
 - Rename DumbCS to LargestFirstCoinSelection
 - Rename must_use_utxos to required_utxos
 - Rename may_use_utxos to optional_uxtos
@@ -529,7 +529,7 @@ final transaction is created by calling `finish` on the builder.
 - Use TXIN_DEFAULT_WEIGHT constant in coin selection
 - Replace `must_use` with `required` in coin selection
 - Take both spending policies into account in create_tx
-- Check last derivation in cache to avoid recomputation
+- Check last derivation in cache to avoid recomputing
 - Use the branch-and-bound cs by default
 - Make coin_select return UTXOs instead of TxIns
 - Build output lookup inside complete transaction
@@ -550,7 +550,7 @@ final transaction is created by calling `finish` on the builder.
 - Require esplora feature for repl example
 
 #### Security
-- Use dirs-next instead of dirs since the latter is unmantained
+- Use dirs-next instead of dirs since the latter is unmaintained
 
 ## [0.1.0-beta.1] - 2020-09-08
 

CONTRIBUTING.md

@@ -46,15 +46,15 @@ Every new feature should be covered by functional tests where possible.
 When refactoring, structure your PR to make it easy to review and don't
 hesitate to split it into multiple small, focused PRs.
 
-The Minimal Supported Rust Version is 1.46 (enforced by our CI).
+The Minimal Supported Rust Version is **1.57.0** (enforced by our CI).
 
 Commits should cover both the issue fixed and the solution's rationale.
-These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind.
+These [guidelines](https://chris.beams.io/posts/git-commit/) should be kept in mind. Commit messages should follow the ["Conventional Commits 1.0.0"](https://www.conventionalcommits.org/en/v1.0.0/) to make commit histories easier to read by humans and automated tools. 
 
 To facilitate communication with other contributors, the project is making use
 of GitHub's "assignee" field. First check that no one is assigned and then
 comment suggesting that you're working on it. If someone is already assigned,
-don't hesitate to ask if the assigned party or previous commenters are still
+don't hesitate to ask if the assigned party or previous commenter are still
 working on it if it has been awhile.
 
 Deprecation policy
@@ -91,7 +91,7 @@ This is also enforced by the CI.
 Security
 --------
 
-Security is a high priority of BDK; disclosure of security vulnerabilites helps
+Security is a high priority of BDK; disclosure of security vulnerabilities helps
 prevent user loss of funds.
 
 Note that BDK is currently considered "pre-production" during this time, there

Cargo.toml

@@ -1,15 +1,23 @@
 [workspace]
+resolver = "2"
 members = [
-    "crates/bdk",
+    "crates/wallet",
     "crates/chain",
     "crates/file_store",
+    "crates/sqlite",
     "crates/electrum",
     "crates/esplora",
+    "crates/bitcoind_rpc",
+    "crates/hwi",
+    "crates/testenv",
     "example-crates/example_cli",
     "example-crates/example_electrum",
+    "example-crates/example_esplora",
+    "example-crates/example_bitcoind_rpc_polling",
     "example-crates/wallet_electrum",
-    "example-crates/wallet_esplora",
+    "example-crates/wallet_esplora_blocking",
     "example-crates/wallet_esplora_async",
+    "example-crates/wallet_rpc",
     "nursery/tmp_plan",
     "nursery/coin_select"
 ]

README.md

@@ -10,19 +10,19 @@
   </p>
 
   <p>
-    <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
+    <a href="https://crates.io/crates/bdk_wallet"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk_wallet.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
-    <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
-    <a href="https://blog.rust-lang.org/2021/12/02/Rust-1.57.0.html"><img alt="Rustc Version 1.57.0+" src="https://img.shields.io/badge/rustc-1.57.0%2B-lightgrey.svg"/></a>
+    <a href="https://docs.rs/bdk_wallet"><img alt="Wallet API Docs" src="https://img.shields.io/badge/docs.rs-bdk_wallet-green"/></a>
+    <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
   <h4>
     <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://docs.rs/bdk">Documentation</a>
+    <a href="https://docs.rs/bdk_wallet">Documentation</a>
   </h4>
 </div>
 
@@ -33,22 +33,60 @@ It is built upon the excellent [`rust-bitcoin`] and [`rust-miniscript`] crates.
 
 > ⚠ The Bitcoin Dev Kit developers are in the process of releasing a `v1.0` which is a fundamental re-write of how the library works.
 > See for some background on this project: https://bitcoindevkit.org/blog/road-to-bdk-1/ (ignore the timeline 😁)
-> For a release timeline see the [`bdk_core_staging`] repo where a lot of the component work is being done. The plan is that everything in the `bdk_core_staging` repo will be moved into the `crates` directory here.
+> For a release timeline see the [`BDK 1.0 project page`].
 
 ## Architecture
 
 The project is split up into several crates in the `/crates` directory:
 
-- [`bdk`](./crates/bdk): Contains the central high level `Wallet` type that is built from the low-level mechanisms provided by the other components
+- [`wallet`](./crates/wallet): Contains the central high level `Wallet` type that is built from the low-level mechanisms provided by the other components
 - [`chain`](./crates/chain): Tools for storing and indexing chain data
+- [`persist`](./crates/persist): Types that define data persistence of a BDK wallet
 - [`file_store`](./crates/file_store): A (experimental) persistence backend for storing chain data in a single file.
 - [`esplora`](./crates/esplora): Extends the [`esplora-client`] crate with methods to fetch chain data from an esplora HTTP server in the form that [`bdk_chain`] and `Wallet` can consume.
 - [`electrum`](./crates/electrum): Extends the [`electrum-client`] crate with methods to fetch chain data from an electrum server in the form that [`bdk_chain`] and `Wallet` can consume.
 
-Fully working examples of how to use these components are in `/example-crates`
+Fully working examples of how to use these components are in `/example-crates`:
+- [`example_cli`](./example-crates/example_cli): Library used by the `example_*` crates. Provides utilities for syncing, showing the balance, generating addresses and creating transactions without using the bdk_wallet `Wallet`.
+- [`example_electrum`](./example-crates/example_electrum): A command line Bitcoin wallet application built on top of `example_cli` and the `electrum` crate. It shows the power of the bdk tools (`chain` + `file_store` + `electrum`), without depending on the main `bdk_wallet` library.
+- [`example_esplora`](./example-crates/example_esplora): A command line Bitcoin wallet application built on top of `example_cli` and the `esplora` crate. It shows the power of the bdk tools (`chain` + `file_store` + `esplora`), without depending on the main `bdk_wallet` library.
+- [`example_bitcoind_rpc_polling`](./example-crates/example_bitcoind_rpc_polling): A command line Bitcoin wallet application built on top of `example_cli` and the `bitcoind_rpc` crate. It shows the power of the bdk tools (`chain` + `file_store` + `bitcoind_rpc`), without depending on the main `bdk_wallet` library.
+- [`wallet_esplora_blocking`](./example-crates/wallet_esplora_blocking): Uses the `Wallet` to sync and spend using the Esplora blocking interface.
+- [`wallet_esplora_async`](./example-crates/wallet_esplora_async): Uses the `Wallet` to sync and spend using the Esplora asynchronous interface.
+- [`wallet_electrum`](./example-crates/wallet_electrum): Uses the `Wallet` to sync and spend using Electrum.
 
-[`bdk_core_staging`]: https://github.com/LLFourn/bdk_core_staging
+[`BDK 1.0 project page`]: https://github.com/orgs/bitcoindevkit/projects/14
 [`rust-miniscript`]: https://github.com/rust-bitcoin/rust-miniscript
 [`rust-bitcoin`]: https://github.com/rust-bitcoin/rust-bitcoin
-[`esplora-client`]: https://docs.rs/esplora-client/0.3.0/esplora_client/
-[`electrum-client`]: https://docs.rs/electrum-client/0.13.0/electrum_client/
+[`esplora-client`]: https://docs.rs/esplora-client/
+[`electrum-client`]: https://docs.rs/electrum-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/
+
+## Minimum Supported Rust Version (MSRV)
+This library should compile with any combination of features with Rust 1.63.0.
+
+To build with the MSRV you will need to pin dependencies as follows:
+
+```shell
+cargo update -p zstd-sys --precise "2.0.8+zstd.1.5.5"
+cargo update -p time --precise "0.3.20"
+cargo update -p home --precise "0.5.5"
+cargo update -p proptest --precise "1.2.0"
+cargo update -p url --precise "2.5.0"
+```
+
+## License
+
+Licensed under either of
+
+* Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
+* MIT license ([LICENSE-MIT](LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
+
+at your option.
+
+### Contribution
+
+Unless you explicitly state otherwise, any contribution intentionally
+submitted for inclusion in the work by you, as defined in the Apache-2.0
+license, shall be dual licensed as above, without any additional terms or
+conditions.

clippy.toml

@@ -1 +1 @@
-msrv="1.57.0"
+msrv="1.63.0"

crates/bdk/examples/compiler.rs

@@ -1,73 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-extern crate bdk;
-extern crate bitcoin;
-extern crate log;
-extern crate miniscript;
-extern crate serde_json;
-
-use std::error::Error;
-use std::str::FromStr;
-
-use log::info;
-
-use bitcoin::Network;
-use miniscript::policy::Concrete;
-use miniscript::Descriptor;
-
-use bdk::wallet::AddressIndex::New;
-use bdk::{KeychainKind, Wallet};
-
-/// Miniscript policy is a high level abstraction of spending conditions. Defined in the
-/// rust-miniscript library here  https://docs.rs/miniscript/7.0.0/miniscript/policy/index.html
-/// rust-miniscript provides a `compile()` function that can be used to compile any miniscript policy
-/// into a descriptor. This descriptor then in turn can be used in bdk a fully functioning wallet
-/// can be derived from the policy.
-///
-/// This example demonstrates the interaction between a bdk wallet and miniscript policy.
-
-fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
-    // We start with a generic miniscript policy string
-    let policy_str = "or(10@thresh(4,pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2)),1@and(older(4209713),thresh(2,pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),pk(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),pk(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068))))";
-    info!("Compiling policy: \n{}", policy_str);
-
-    // Parse the string as a [`Concrete`] type miniscript policy.
-    let policy = Concrete::<String>::from_str(policy_str)?;
-
-    // Create a `wsh` type descriptor from the policy.
-    // `policy.compile()` returns the resulting miniscript from the policy.
-    let descriptor = Descriptor::new_wsh(policy.compile()?)?;
-
-    info!("Compiled into following Descriptor: \n{}", descriptor);
-
-    // Create a new wallet from this descriptor
-    let mut wallet = Wallet::new_no_persist(&format!("{}", descriptor), None, Network::Regtest)?;
-
-    info!(
-        "First derived address from the descriptor: \n{}",
-        wallet.get_address(New)
-    );
-
-    // BDK also has it's own `Policy` structure to represent the spending condition in a more
-    // human readable json format.
-    let spending_policy = wallet.policies(KeychainKind::External)?;
-    info!(
-        "The BDK spending policy: \n{}",
-        serde_json::to_string_pretty(&spending_policy)?
-    );
-
-    Ok(())
-}

crates/bdk/src/error.rs

@@ -1,201 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-use crate::bitcoin::Network;
-use crate::{descriptor, wallet};
-use alloc::{string::String, vec::Vec};
-use bitcoin::{OutPoint, Txid};
-use core::fmt;
-
-/// Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
-#[derive(Debug)]
-pub enum Error {
-    /// Generic error
-    Generic(String),
-    /// Cannot build a tx without recipients
-    NoRecipients,
-    /// `manually_selected_only` option is selected but no utxo has been passed
-    NoUtxosSelected,
-    /// Output created is under the dust limit, 546 satoshis
-    OutputBelowDustLimit(usize),
-    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
-    InsufficientFunds {
-        /// Sats needed for some transaction
-        needed: u64,
-        /// Sats available for spending
-        available: u64,
-    },
-    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
-    /// exponentially, thus a limit is set, and when hit, this error is thrown
-    BnBTotalTriesExceeded,
-    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
-    /// the desired outputs plus fee, if there is not such combination this error is thrown
-    BnBNoExactMatch,
-    /// Happens when trying to spend an UTXO that is not in the internal database
-    UnknownUtxo,
-    /// Thrown when a tx is not found in the internal database
-    TransactionNotFound,
-    /// Happens when trying to bump a transaction that is already confirmed
-    TransactionConfirmed,
-    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
-    IrreplaceableTransaction,
-    /// When bumping a tx the fee rate requested is lower than required
-    FeeRateTooLow {
-        /// Required fee rate (satoshi/vbyte)
-        required: crate::types::FeeRate,
-    },
-    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
-    FeeTooLow {
-        /// Required fee absolute value (satoshi)
-        required: u64,
-    },
-    /// Node doesn't have data to estimate a fee rate
-    FeeRateUnavailable,
-    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
-    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
-    /// explicit origin provided
-    ///
-    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
-    MissingKeyOrigin(String),
-    /// Error while working with [`keys`](crate::keys)
-    Key(crate::keys::KeyError),
-    /// Descriptor checksum mismatch
-    ChecksumMismatch,
-    /// Spending policy is not compatible with this [`KeychainKind`](crate::types::KeychainKind)
-    SpendingPolicyRequired(crate::types::KeychainKind),
-    /// Error while extracting and manipulating policies
-    InvalidPolicyPathError(crate::descriptor::policy::PolicyError),
-    /// Signing error
-    Signer(crate::wallet::signer::SignerError),
-    /// Requested outpoint doesn't exist in the tx (vout greater than available outputs)
-    InvalidOutpoint(OutPoint),
-    /// Error related to the parsing and usage of descriptors
-    Descriptor(crate::descriptor::error::Error),
-    /// Miniscript error
-    Miniscript(miniscript::Error),
-    /// Miniscript PSBT error
-    MiniscriptPsbt(MiniscriptPsbtError),
-    /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
-    /// Partially signed bitcoin transaction error
-    Psbt(bitcoin::util::psbt::Error),
-}
-
-/// Errors returned by miniscript when updating inconsistent PSBTs
-#[derive(Debug, Clone)]
-pub enum MiniscriptPsbtError {
-    Conversion(miniscript::descriptor::ConversionError),
-    UtxoUpdate(miniscript::psbt::UtxoUpdateError),
-    OutputUpdate(miniscript::psbt::OutputUpdateError),
-}
-
-impl fmt::Display for MiniscriptPsbtError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Conversion(err) => write!(f, "Conversion error: {}", err),
-            Self::UtxoUpdate(err) => write!(f, "UTXO update error: {}", err),
-            Self::OutputUpdate(err) => write!(f, "Output update error: {}", err),
-        }
-    }
-}
-
-#[cfg(feature = "std")]
-impl std::error::Error for MiniscriptPsbtError {}
-
-#[cfg(feature = "std")]
-impl fmt::Display for Error {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            Self::Generic(err) => write!(f, "Generic error: {}", err),
-            Self::NoRecipients => write!(f, "Cannot build tx without recipients"),
-            Self::NoUtxosSelected => write!(f, "No UTXO selected"),
-            Self::OutputBelowDustLimit(limit) => {
-                write!(f, "Output below the dust limit: {}", limit)
-            }
-            Self::InsufficientFunds { needed, available } => write!(
-                f,
-                "Insufficient funds: {} sat available of {} sat needed",
-                available, needed
-            ),
-            Self::BnBTotalTriesExceeded => {
-                write!(f, "Branch and bound coin selection: total tries exceeded")
-            }
-            Self::BnBNoExactMatch => write!(f, "Branch and bound coin selection: not exact match"),
-            Self::UnknownUtxo => write!(f, "UTXO not found in the internal database"),
-            Self::TransactionNotFound => {
-                write!(f, "Transaction not found in the internal database")
-            }
-            Self::TransactionConfirmed => write!(f, "Transaction already confirmed"),
-            Self::IrreplaceableTransaction => write!(f, "Transaction can't be replaced"),
-            Self::FeeRateTooLow { required } => write!(
-                f,
-                "Fee rate too low: required {} sat/vbyte",
-                required.as_sat_per_vb()
-            ),
-            Self::FeeTooLow { required } => write!(f, "Fee to low: required {} sat", required),
-            Self::FeeRateUnavailable => write!(f, "Fee rate unavailable"),
-            Self::MissingKeyOrigin(err) => write!(f, "Missing key origin: {}", err),
-            Self::Key(err) => write!(f, "Key error: {}", err),
-            Self::ChecksumMismatch => write!(f, "Descriptor checksum mismatch"),
-            Self::SpendingPolicyRequired(keychain_kind) => {
-                write!(f, "Spending policy required: {:?}", keychain_kind)
-            }
-            Self::InvalidPolicyPathError(err) => write!(f, "Invalid policy path: {}", err),
-            Self::Signer(err) => write!(f, "Signer error: {}", err),
-            Self::InvalidOutpoint(outpoint) => write!(
-                f,
-                "Requested outpoint doesn't exist in the tx: {}",
-                outpoint
-            ),
-            Self::Descriptor(err) => write!(f, "Descriptor error: {}", err),
-            Self::Miniscript(err) => write!(f, "Miniscript error: {}", err),
-            Self::MiniscriptPsbt(err) => write!(f, "Miniscript PSBT error: {}", err),
-            Self::Bip32(err) => write!(f, "BIP32 error: {}", err),
-            Self::Psbt(err) => write!(f, "PSBT error: {}", err),
-        }
-    }
-}
-
-#[cfg(feature = "std")]
-impl std::error::Error for Error {}
-
-macro_rules! impl_error {
-    ( $from:ty, $to:ident ) => {
-        impl_error!($from, $to, Error);
-    };
-    ( $from:ty, $to:ident, $impl_for:ty ) => {
-        impl core::convert::From<$from> for $impl_for {
-            fn from(err: $from) -> Self {
-                <$impl_for>::$to(err)
-            }
-        }
-    };
-}
-
-impl_error!(descriptor::error::Error, Descriptor);
-impl_error!(descriptor::policy::PolicyError, InvalidPolicyPathError);
-impl_error!(wallet::signer::SignerError, Signer);
-
-impl From<crate::keys::KeyError> for Error {
-    fn from(key_error: crate::keys::KeyError) -> Error {
-        match key_error {
-            crate::keys::KeyError::Miniscript(inner) => Error::Miniscript(inner),
-            crate::keys::KeyError::Bip32(inner) => Error::Bip32(inner),
-            crate::keys::KeyError::InvalidChecksum => Error::ChecksumMismatch,
-            e => Error::Key(e),
-        }
-    }
-}
-
-impl_error!(miniscript::Error, Miniscript);
-impl_error!(MiniscriptPsbtError, MiniscriptPsbt);
-impl_error!(bitcoin::util::bip32::Error, Bip32);
-impl_error!(bitcoin::util::psbt::Error, Psbt);

crates/bdk/src/types.rs

@@ -1,339 +0,0 @@
-// Bitcoin Dev Kit
-// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
-//
-// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
-//
-// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
-// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
-// You may not use this file except in accordance with one or both of these
-// licenses.
-
-use alloc::boxed::Box;
-use core::convert::AsRef;
-use core::ops::Sub;
-
-use bdk_chain::ConfirmationTime;
-use bitcoin::blockdata::transaction::{OutPoint, Transaction, TxOut};
-use bitcoin::{hash_types::Txid, util::psbt};
-
-use serde::{Deserialize, Serialize};
-
-/// Types of keychains
-#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
-pub enum KeychainKind {
-    /// External keychain, used for deriving recipient addresses.
-    External = 0,
-    /// Internal keychain, used for deriving change addresses.
-    Internal = 1,
-}
-
-impl KeychainKind {
-    /// Return [`KeychainKind`] as a byte
-    pub fn as_byte(&self) -> u8 {
-        match self {
-            KeychainKind::External => b'e',
-            KeychainKind::Internal => b'i',
-        }
-    }
-}
-
-impl AsRef<[u8]> for KeychainKind {
-    fn as_ref(&self) -> &[u8] {
-        match self {
-            KeychainKind::External => b"e",
-            KeychainKind::Internal => b"i",
-        }
-    }
-}
-
-/// Fee rate
-#[derive(Debug, Copy, Clone, PartialEq, PartialOrd)]
-// Internally stored as satoshi/vbyte
-pub struct FeeRate(f32);
-
-impl FeeRate {
-    /// Create a new instance checking the value provided
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    fn new_checked(value: f32) -> Self {
-        assert!(value.is_normal() || value == 0.0);
-        assert!(value.is_sign_positive());
-
-        FeeRate(value)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in sats/kwu
-    pub fn from_sat_per_kwu(sat_per_kwu: f32) -> Self {
-        FeeRate::new_checked(sat_per_kwu / 250.0_f32)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in sats/kvb
-    pub fn from_sat_per_kvb(sat_per_kvb: f32) -> Self {
-        FeeRate::new_checked(sat_per_kvb / 1000.0_f32)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in btc/kvbytes
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    pub fn from_btc_per_kvb(btc_per_kvb: f32) -> Self {
-        FeeRate::new_checked(btc_per_kvb * 1e5)
-    }
-
-    /// Create a new instance of [`FeeRate`] given a float fee rate in satoshi/vbyte
-    ///
-    /// ## Panics
-    ///
-    /// Panics if the value is not [normal](https://doc.rust-lang.org/std/primitive.f32.html#method.is_normal) (except if it's a positive zero) or negative.
-    pub fn from_sat_per_vb(sat_per_vb: f32) -> Self {
-        FeeRate::new_checked(sat_per_vb)
-    }
-
-    /// Create a new [`FeeRate`] with the default min relay fee value
-    pub const fn default_min_relay_fee() -> Self {
-        FeeRate(1.0)
-    }
-
-    /// Calculate fee rate from `fee` and weight units (`wu`).
-    pub fn from_wu(fee: u64, wu: usize) -> FeeRate {
-        Self::from_vb(fee, wu.vbytes())
-    }
-
-    /// Calculate fee rate from `fee` and `vbytes`.
-    pub fn from_vb(fee: u64, vbytes: usize) -> FeeRate {
-        let rate = fee as f32 / vbytes as f32;
-        Self::from_sat_per_vb(rate)
-    }
-
-    /// Return the value as satoshi/vbyte
-    pub fn as_sat_per_vb(&self) -> f32 {
-        self.0
-    }
-
-    /// Return the value as satoshi/kwu
-    pub fn sat_per_kwu(&self) -> f32 {
-        self.0 * 250.0_f32
-    }
-
-    /// Calculate absolute fee in Satoshis using size in weight units.
-    pub fn fee_wu(&self, wu: usize) -> u64 {
-        self.fee_vb(wu.vbytes())
-    }
-
-    /// Calculate absolute fee in Satoshis using size in virtual bytes.
-    pub fn fee_vb(&self, vbytes: usize) -> u64 {
-        (self.as_sat_per_vb() * vbytes as f32).ceil() as u64
-    }
-}
-
-impl Default for FeeRate {
-    fn default() -> Self {
-        FeeRate::default_min_relay_fee()
-    }
-}
-
-impl Sub for FeeRate {
-    type Output = Self;
-
-    fn sub(self, other: FeeRate) -> Self::Output {
-        FeeRate(self.0 - other.0)
-    }
-}
-
-/// Trait implemented by types that can be used to measure weight units.
-pub trait Vbytes {
-    /// Convert weight units to virtual bytes.
-    fn vbytes(self) -> usize;
-}
-
-impl Vbytes for usize {
-    fn vbytes(self) -> usize {
-        // ref: https://github.com/bitcoin/bips/blob/master/bip-0141.mediawiki#transaction-size-calculations
-        (self as f32 / 4.0).ceil() as usize
-    }
-}
-
-/// An unspent output owned by a [`Wallet`].
-///
-/// [`Wallet`]: crate::Wallet
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
-pub struct LocalUtxo {
-    /// Reference to a transaction output
-    pub outpoint: OutPoint,
-    /// Transaction output
-    pub txout: TxOut,
-    /// Type of keychain
-    pub keychain: KeychainKind,
-    /// Whether this UTXO is spent or not
-    pub is_spent: bool,
-    /// The derivation index for the script pubkey in the wallet
-    pub derivation_index: u32,
-    /// The confirmation time for transaction containing this utxo
-    pub confirmation_time: ConfirmationTime,
-}
-
-/// A [`Utxo`] with its `satisfaction_weight`.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct WeightedUtxo {
-    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
-    /// properly maintain the feerate when adding this input to a transaction during coin selection.
-    ///
-    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
-    pub satisfaction_weight: usize,
-    /// The UTXO
-    pub utxo: Utxo,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-/// An unspent transaction output (UTXO).
-pub enum Utxo {
-    /// A UTXO owned by the local wallet.
-    Local(LocalUtxo),
-    /// A UTXO owned by another wallet.
-    Foreign {
-        /// The location of the output.
-        outpoint: OutPoint,
-        /// The information about the input we require to add it to a PSBT.
-        // Box it to stop the type being too big.
-        psbt_input: Box<psbt::Input>,
-    },
-}
-
-impl Utxo {
-    /// Get the location of the UTXO
-    pub fn outpoint(&self) -> OutPoint {
-        match &self {
-            Utxo::Local(local) => local.outpoint,
-            Utxo::Foreign { outpoint, .. } => *outpoint,
-        }
-    }
-
-    /// Get the `TxOut` of the UTXO
-    pub fn txout(&self) -> &TxOut {
-        match &self {
-            Utxo::Local(local) => &local.txout,
-            Utxo::Foreign {
-                outpoint,
-                psbt_input,
-            } => {
-                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
-                    return &prev_tx.output[outpoint.vout as usize];
-                }
-
-                if let Some(txout) = &psbt_input.witness_utxo {
-                    return txout;
-                }
-
-                unreachable!("Foreign UTXOs will always have one of these set")
-            }
-        }
-    }
-}
-
-/// A wallet transaction
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct TransactionDetails {
-    /// Optional transaction
-    pub transaction: Option<Transaction>,
-    /// Transaction id
-    pub txid: Txid,
-    /// Received value (sats)
-    /// Sum of owned outputs of this transaction.
-    pub received: u64,
-    /// Sent value (sats)
-    /// Sum of owned inputs of this transaction.
-    pub sent: u64,
-    /// Fee value in sats if it was available.
-    pub fee: Option<u64>,
-    /// If the transaction is confirmed, contains height and Unix timestamp of the block containing the
-    /// transaction, unconfirmed transaction contains `None`.
-    pub confirmation_time: ConfirmationTime,
-}
-
-impl PartialOrd for TransactionDetails {
-    fn partial_cmp(&self, other: &Self) -> Option<core::cmp::Ordering> {
-        Some(self.cmp(other))
-    }
-}
-
-impl Ord for TransactionDetails {
-    fn cmp(&self, other: &Self) -> core::cmp::Ordering {
-        self.confirmation_time
-            .cmp(&other.confirmation_time)
-            .then_with(|| self.txid.cmp(&other.txid))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn can_store_feerate_in_const() {
-        const _MIN_RELAY: FeeRate = FeeRate::default_min_relay_fee();
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_neg_zero() {
-        let _ = FeeRate::from_sat_per_vb(-0.0);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_neg_value() {
-        let _ = FeeRate::from_sat_per_vb(-5.0);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_nan() {
-        let _ = FeeRate::from_sat_per_vb(f32::NAN);
-    }
-
-    #[test]
-    #[should_panic]
-    fn test_invalid_feerate_inf() {
-        let _ = FeeRate::from_sat_per_vb(f32::INFINITY);
-    }
-
-    #[test]
-    fn test_valid_feerate_pos_zero() {
-        let _ = FeeRate::from_sat_per_vb(0.0);
-    }
-
-    #[test]
-    fn test_fee_from_btc_per_kvb() {
-        let fee = FeeRate::from_btc_per_kvb(1e-5);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_vbyte() {
-        let fee = FeeRate::from_sat_per_vb(1.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_default_min_relay_fee() {
-        let fee = FeeRate::default_min_relay_fee();
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_kvb() {
-        let fee = FeeRate::from_sat_per_kvb(1000.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-    }
-
-    #[test]
-    fn test_fee_from_sat_per_kwu() {
-        let fee = FeeRate::from_sat_per_kwu(250.0);
-        assert!((fee.as_sat_per_vb() - 1.0).abs() < f32::EPSILON);
-        assert_eq!(fee.sat_per_kwu(), 250.0);
-    }
-}

crates/bdk/tests/common.rs

@@ -1,93 +0,0 @@
-#![allow(unused)]
-use bdk::{wallet::AddressIndex, Wallet};
-use bdk_chain::{BlockId, ConfirmationTime};
-use bitcoin::hashes::Hash;
-use bitcoin::{BlockHash, Network, Transaction, TxOut};
-
-/// Return a fake wallet that appears to be funded for testing.
-pub fn get_funded_wallet_with_change(
-    descriptor: &str,
-    change: Option<&str>,
-) -> (Wallet, bitcoin::Txid) {
-    let mut wallet = Wallet::new_no_persist(descriptor, change, Network::Regtest).unwrap();
-    let address = wallet.get_address(AddressIndex::New).address;
-
-    let tx = Transaction {
-        version: 1,
-        lock_time: bitcoin::PackedLockTime(0),
-        input: vec![],
-        output: vec![TxOut {
-            value: 50_000,
-            script_pubkey: address.script_pubkey(),
-        }],
-    };
-
-    wallet
-        .insert_checkpoint(BlockId {
-            height: 1_000,
-            hash: BlockHash::all_zeros(),
-        })
-        .unwrap();
-    wallet
-        .insert_tx(
-            tx.clone(),
-            ConfirmationTime::Confirmed {
-                height: 1_000,
-                time: 100,
-            },
-        )
-        .unwrap();
-
-    (wallet, tx.txid())
-}
-
-pub fn get_funded_wallet(descriptor: &str) -> (Wallet, bitcoin::Txid) {
-    get_funded_wallet_with_change(descriptor, None)
-}
-
-pub fn get_test_wpkh() -> &'static str {
-    "wpkh(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW)"
-}
-
-pub fn get_test_single_sig_csv() -> &'static str {
-    // and(pk(Alice),older(6))
-    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),older(6)))"
-}
-
-pub fn get_test_a_or_b_plus_csv() -> &'static str {
-    // or(pk(Alice),and(pk(Bob),older(144)))
-    "wsh(or_d(pk(cRjo6jqfVNP33HhSS76UhXETZsGTZYx8FMFvR9kpbtCSV1PmdZdu),and_v(v:pk(cMnkdebixpXMPfkcNEjjGin7s94hiehAH4mLbYkZoh9KSiNNmqC8),older(144))))"
-}
-
-pub fn get_test_single_sig_cltv() -> &'static str {
-    // and(pk(Alice),after(100000))
-    "wsh(and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100000)))"
-}
-
-pub fn get_test_tr_single_sig() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG)"
-}
-
-pub fn get_test_tr_with_taptree() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}
-
-pub fn get_test_tr_with_taptree_both_priv() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{pk(cPZzKuNmpuUjD1e8jUU4PVzy2b5LngbSip8mBsxf4e7rSFZVb4Uh),pk(cNaQCDwmmh4dS9LzCgVtyy1e1xjCJ21GUDHe9K98nzb689JvinGV)})"
-}
-
-pub fn get_test_tr_repeated_key() -> &'static str {
-    "tr(b511bd5771e47ee27558b1765e87b541668304ec567721c7b880edc0a010da55,{and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(100)),and_v(v:pk(cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW),after(200))})"
-}
-
-pub fn get_test_tr_single_sig_xprv() -> &'static str {
-    "tr(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*)"
-}
-
-pub fn get_test_tr_with_taptree_xprv() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(tprv8ZgxMBicQKsPdDArR4xSAECuVxeX1jwwSXR4ApKbkYgZiziDc4LdBy2WvJeGDfUSE4UT4hHhbgEwbdq8ajjUHiKDegkwrNU6V55CxcxonVN/*),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}
-
-pub fn get_test_tr_dup_keys() -> &'static str {
-    "tr(cNJmN3fH9DDbDt131fQNkVakkpzawJBSeybCUNmP1BovpmGQ45xG,{pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642),pk(8aee2b8120a5f157f1223f72b5e62b825831a27a9fdf427db7cc697494d4a642)})"
-}

crates/bdk/tests/psbt.rs

@@ -1,158 +0,0 @@
-use bdk::bitcoin::TxIn;
-use bdk::wallet::AddressIndex;
-use bdk::wallet::AddressIndex::New;
-use bdk::{psbt, FeeRate, SignOptions};
-use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
-use core::str::FromStr;
-mod common;
-use common::*;
-
-// from bip 174
-const PSBT_STR: &str = "cHNidP8BAKACAAAAAqsJSaCMWvfEm4IS9Bfi8Vqz9cM9zxU4IagTn4d6W3vkAAAAAAD+////qwlJoIxa98SbghL0F+LxWrP1wz3PFTghqBOfh3pbe+QBAAAAAP7///8CYDvqCwAAAAAZdqkUdopAu9dAy+gdmI5x3ipNXHE5ax2IrI4kAAAAAAAAGXapFG9GILVT+glechue4O/p+gOcykWXiKwAAAAAAAEHakcwRAIgR1lmF5fAGwNrJZKJSGhiGDR9iYZLcZ4ff89X0eURZYcCIFMJ6r9Wqk2Ikf/REf3xM286KdqGbX+EhtdVRs7tr5MZASEDXNxh/HupccC1AaZGoqg7ECy0OIEhfKaC3Ibi1z+ogpIAAQEgAOH1BQAAAAAXqRQ1RebjO4MsRwUPJNPuuTycA5SLx4cBBBYAFIXRNTfy4mVAWjTbr6nj3aAfuCMIAAAA";
-
-#[test]
-#[should_panic(expected = "InputIndexOutOfRange")]
-fn test_psbt_malformed_psbt_input_legacy() {
-    let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
-    let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
-    let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
-    psbt.inputs.push(psbt_bip.inputs[0].clone());
-    let options = SignOptions {
-        trust_witness_utxo: true,
-        ..Default::default()
-    };
-    let _ = wallet.sign(&mut psbt, options).unwrap();
-}
-
-#[test]
-#[should_panic(expected = "InputIndexOutOfRange")]
-fn test_psbt_malformed_psbt_input_segwit() {
-    let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
-    let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
-    let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
-    psbt.inputs.push(psbt_bip.inputs[1].clone());
-    let options = SignOptions {
-        trust_witness_utxo: true,
-        ..Default::default()
-    };
-    let _ = wallet.sign(&mut psbt, options).unwrap();
-}
-
-#[test]
-#[should_panic(expected = "InputIndexOutOfRange")]
-fn test_psbt_malformed_tx_input() {
-    let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
-    let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
-    psbt.unsigned_tx.input.push(TxIn::default());
-    let options = SignOptions {
-        trust_witness_utxo: true,
-        ..Default::default()
-    };
-    let _ = wallet.sign(&mut psbt, options).unwrap();
-}
-
-#[test]
-fn test_psbt_sign_with_finalized() {
-    let psbt_bip = Psbt::from_str(PSBT_STR).unwrap();
-    let (mut wallet, _) = get_funded_wallet(get_test_wpkh());
-    let send_to = wallet.get_address(AddressIndex::New);
-    let mut builder = wallet.build_tx();
-    builder.add_recipient(send_to.script_pubkey(), 10_000);
-    let (mut psbt, _) = builder.finish().unwrap();
-
-    // add a finalized input
-    psbt.inputs.push(psbt_bip.inputs[0].clone());
-    psbt.unsigned_tx
-        .input
-        .push(psbt_bip.unsigned_tx.input[0].clone());
-
-    let _ = wallet.sign(&mut psbt, SignOptions::default()).unwrap();
-}
-
-#[test]
-fn test_psbt_fee_rate_with_witness_utxo() {
-    use psbt::PsbtUtils;
-
-    let expected_fee_rate = 1.2345;
-
-    let (mut wallet, _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wallet.get_address(New);
-    let mut builder = wallet.build_tx();
-    builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut psbt, _) = builder.finish().unwrap();
-    let fee_amount = psbt.fee_amount();
-    assert!(fee_amount.is_some());
-
-    let unfinalized_fee_rate = psbt.fee_rate().unwrap();
-
-    let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
-    assert!(finalized);
-
-    let finalized_fee_rate = psbt.fee_rate().unwrap();
-    assert!(finalized_fee_rate.as_sat_per_vb() >= expected_fee_rate);
-    assert!(finalized_fee_rate.as_sat_per_vb() < unfinalized_fee_rate.as_sat_per_vb());
-}
-
-#[test]
-fn test_psbt_fee_rate_with_nonwitness_utxo() {
-    use psbt::PsbtUtils;
-
-    let expected_fee_rate = 1.2345;
-
-    let (mut wallet, _) = get_funded_wallet("pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wallet.get_address(New);
-    let mut builder = wallet.build_tx();
-    builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut psbt, _) = builder.finish().unwrap();
-    let fee_amount = psbt.fee_amount();
-    assert!(fee_amount.is_some());
-    let unfinalized_fee_rate = psbt.fee_rate().unwrap();
-
-    let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
-    assert!(finalized);
-
-    let finalized_fee_rate = psbt.fee_rate().unwrap();
-    assert!(finalized_fee_rate.as_sat_per_vb() >= expected_fee_rate);
-    assert!(finalized_fee_rate.as_sat_per_vb() < unfinalized_fee_rate.as_sat_per_vb());
-}
-
-#[test]
-fn test_psbt_fee_rate_with_missing_txout() {
-    use psbt::PsbtUtils;
-
-    let expected_fee_rate = 1.2345;
-
-    let (mut wpkh_wallet,  _) = get_funded_wallet("wpkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = wpkh_wallet.get_address(New);
-    let mut builder = wpkh_wallet.build_tx();
-    builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut wpkh_psbt, _) = builder.finish().unwrap();
-
-    wpkh_psbt.inputs[0].witness_utxo = None;
-    wpkh_psbt.inputs[0].non_witness_utxo = None;
-    assert!(wpkh_psbt.fee_amount().is_none());
-    assert!(wpkh_psbt.fee_rate().is_none());
-
-    let (mut pkh_wallet,  _) = get_funded_wallet("pkh(tprv8ZgxMBicQKsPd3EupYiPRhaMooHKUHJxNsTfYuScep13go8QFfHdtkG9nRkFGb7busX4isf6X9dURGCoKgitaApQ6MupRhZMcELAxTBRJgS/*)");
-    let addr = pkh_wallet.get_address(New);
-    let mut builder = pkh_wallet.build_tx();
-    builder.drain_to(addr.script_pubkey()).drain_wallet();
-    builder.fee_rate(FeeRate::from_sat_per_vb(expected_fee_rate));
-    let (mut pkh_psbt, _) = builder.finish().unwrap();
-
-    pkh_psbt.inputs[0].non_witness_utxo = None;
-    assert!(pkh_psbt.fee_amount().is_none());
-    assert!(pkh_psbt.fee_rate().is_none());
-}

crates/bitcoind_rpc/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "bdk_bitcoind_rpc"
+version = "0.12.0"
+edition = "2021"
+rust-version = "1.63"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_bitcoind_rpc"
+description = "This crate is used for emitting blockchain data from the `bitcoind` RPC interface."
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bitcoin = { version = "0.32.0", default-features = false }
+bitcoincore-rpc = { version = "0.19.0" }
+bdk_chain = { path = "../chain", version = "0.16", default-features = false }
+
+[dev-dependencies]
+bdk_testenv = { path = "../testenv", default-features = false }
+
+[features]
+default = ["std"]
+std = ["bitcoin/std", "bdk_chain/std"]
+serde = ["bitcoin/serde", "bdk_chain/serde"]

crates/bitcoind_rpc/README.md

@@ -0,0 +1,3 @@
+# BDK Bitcoind RPC
+
+This crate is used for emitting blockchain data from the `bitcoind` RPC interface.

crates/bitcoind_rpc/src/lib.rs

@@ -0,0 +1,328 @@
+//! This crate is used for emitting blockchain data from the `bitcoind` RPC interface. It does not
+//! use the wallet RPC API, so this crate can be used with wallet-disabled Bitcoin Core nodes.
+//!
+//! [`Emitter`] is the main structure which sources blockchain data from [`bitcoincore_rpc::Client`].
+//!
+//! To only get block updates (exclude mempool transactions), the caller can use
+//! [`Emitter::next_block`] or/and [`Emitter::next_header`] until it returns `Ok(None)` (which means
+//! the chain tip is reached). A separate method, [`Emitter::mempool`] can be used to emit the whole
+//! mempool.
+#![warn(missing_docs)]
+
+use bdk_chain::{local_chain::CheckPoint, BlockId};
+use bitcoin::{block::Header, Block, BlockHash, Transaction};
+pub use bitcoincore_rpc;
+use bitcoincore_rpc::bitcoincore_rpc_json;
+
+/// The [`Emitter`] is used to emit data sourced from [`bitcoincore_rpc::Client`].
+///
+/// Refer to [module-level documentation] for more.
+///
+/// [module-level documentation]: crate
+pub struct Emitter<'c, C> {
+    client: &'c C,
+    start_height: u32,
+
+    /// The checkpoint of the last-emitted block that is in the best chain. If it is later found
+    /// that the block is no longer in the best chain, it will be popped off from here.
+    last_cp: CheckPoint,
+
+    /// The block result returned from rpc of the last-emitted block. As this result contains the
+    /// next block's block hash (which we use to fetch the next block), we set this to `None`
+    /// whenever there are no more blocks, or the next block is no longer in the best chain. This
+    /// gives us an opportunity to re-fetch this result.
+    last_block: Option<bitcoincore_rpc_json::GetBlockResult>,
+
+    /// The latest first-seen epoch of emitted mempool transactions. This is used to determine
+    /// whether a mempool transaction is already emitted.
+    last_mempool_time: usize,
+
+    /// The last emitted block during our last mempool emission. This is used to determine whether
+    /// there has been a reorg since our last mempool emission.
+    last_mempool_tip: Option<u32>,
+}
+
+impl<'c, C: bitcoincore_rpc::RpcApi> Emitter<'c, C> {
+    /// Construct a new [`Emitter`].
+    ///
+    /// `last_cp` informs the emitter of the chain we are starting off with. This way, the emitter
+    /// can start emission from a block that connects to the original chain.
+    ///
+    /// `start_height` starts emission from a given height (if there are no conflicts with the
+    /// original chain).
+    pub fn new(client: &'c C, last_cp: CheckPoint, start_height: u32) -> Self {
+        Self {
+            client,
+            start_height,
+            last_cp,
+            last_block: None,
+            last_mempool_time: 0,
+            last_mempool_tip: None,
+        }
+    }
+
+    /// Emit mempool transactions, alongside their first-seen unix timestamps.
+    ///
+    /// This method emits each transaction only once, unless we cannot guarantee the transaction's
+    /// ancestors are already emitted.
+    ///
+    /// To understand why, consider a receiver which filters transactions based on whether it
+    /// alters the UTXO set of tracked script pubkeys. If an emitted mempool transaction spends a
+    /// tracked UTXO which is confirmed at height `h`, but the receiver has only seen up to block
+    /// of height `h-1`, we want to re-emit this transaction until the receiver has seen the block
+    /// at height `h`.
+    pub fn mempool(&mut self) -> Result<Vec<(Transaction, u64)>, bitcoincore_rpc::Error> {
+        let client = self.client;
+
+        // This is the emitted tip height during the last mempool emission.
+        let prev_mempool_tip = self
+            .last_mempool_tip
+            // We use `start_height - 1` as we cannot guarantee that the block at
+            // `start_height` has been emitted.
+            .unwrap_or(self.start_height.saturating_sub(1));
+
+        // Mempool txs come with a timestamp of when the tx is introduced to the mempool. We keep
+        // track of the latest mempool tx's timestamp to determine whether we have seen a tx
+        // before. `prev_mempool_time` is the previous timestamp and `last_time` records what will
+        // be the new latest timestamp.
+        let prev_mempool_time = self.last_mempool_time;
+        let mut latest_time = prev_mempool_time;
+
+        let txs_to_emit = client
+            .get_raw_mempool_verbose()?
+            .into_iter()
+            .filter_map({
+                let latest_time = &mut latest_time;
+                move |(txid, tx_entry)| -> Option<Result<_, bitcoincore_rpc::Error>> {
+                    let tx_time = tx_entry.time as usize;
+                    if tx_time > *latest_time {
+                        *latest_time = tx_time;
+                    }
+
+                    // Avoid emitting transactions that are already emitted if we can guarantee
+                    // blocks containing ancestors are already emitted. The bitcoind rpc interface
+                    // provides us with the block height that the tx is introduced to the mempool.
+                    // If we have already emitted the block of height, we can assume that all
+                    // ancestor txs have been processed by the receiver.
+                    let is_already_emitted = tx_time <= prev_mempool_time;
+                    let is_within_height = tx_entry.height <= prev_mempool_tip as _;
+                    if is_already_emitted && is_within_height {
+                        return None;
+                    }
+
+                    let tx = match client.get_raw_transaction(&txid, None) {
+                        Ok(tx) => tx,
+                        // the tx is confirmed or evicted since `get_raw_mempool_verbose`
+                        Err(err) if err.is_not_found_error() => return None,
+                        Err(err) => return Some(Err(err)),
+                    };
+
+                    Some(Ok((tx, tx_time as u64)))
+                }
+            })
+            .collect::<Result<Vec<_>, _>>()?;
+
+        self.last_mempool_time = latest_time;
+        self.last_mempool_tip = Some(self.last_cp.height());
+
+        Ok(txs_to_emit)
+    }
+
+    /// Emit the next block height and header (if any).
+    pub fn next_header(&mut self) -> Result<Option<BlockEvent<Header>>, bitcoincore_rpc::Error> {
+        Ok(poll(self, |hash| self.client.get_block_header(hash))?
+            .map(|(checkpoint, block)| BlockEvent { block, checkpoint }))
+    }
+
+    /// Emit the next block height and block (if any).
+    pub fn next_block(&mut self) -> Result<Option<BlockEvent<Block>>, bitcoincore_rpc::Error> {
+        Ok(poll(self, |hash| self.client.get_block(hash))?
+            .map(|(checkpoint, block)| BlockEvent { block, checkpoint }))
+    }
+}
+
+/// A newly emitted block from [`Emitter`].
+#[derive(Debug)]
+pub struct BlockEvent<B> {
+    /// Either a full [`Block`] or [`Header`] of the new block.
+    pub block: B,
+
+    /// The checkpoint of the new block.
+    ///
+    /// A [`CheckPoint`] is a node of a linked list of [`BlockId`]s. This checkpoint is linked to
+    /// all [`BlockId`]s originally passed in [`Emitter::new`] as well as emitted blocks since then.
+    /// These blocks are guaranteed to be of the same chain.
+    ///
+    /// This is important as BDK structures require block-to-apply to be connected with another
+    /// block in the original chain.
+    pub checkpoint: CheckPoint,
+}
+
+impl<B> BlockEvent<B> {
+    /// The block height of this new block.
+    pub fn block_height(&self) -> u32 {
+        self.checkpoint.height()
+    }
+
+    /// The block hash of this new block.
+    pub fn block_hash(&self) -> BlockHash {
+        self.checkpoint.hash()
+    }
+
+    /// The [`BlockId`] of a previous block that this block connects to.
+    ///
+    /// This either returns a [`BlockId`] of a previously emitted block or from the chain we started
+    /// with (passed in as `last_cp` in [`Emitter::new`]).
+    ///
+    /// This value is derived from [`BlockEvent::checkpoint`].
+    pub fn connected_to(&self) -> BlockId {
+        match self.checkpoint.prev() {
+            Some(prev_cp) => prev_cp.block_id(),
+            // there is no previous checkpoint, so just connect with itself
+            None => self.checkpoint.block_id(),
+        }
+    }
+}
+
+enum PollResponse {
+    Block(bitcoincore_rpc_json::GetBlockResult),
+    NoMoreBlocks,
+    /// Fetched block is not in the best chain.
+    BlockNotInBestChain,
+    AgreementFound(bitcoincore_rpc_json::GetBlockResult, CheckPoint),
+    /// Force the genesis checkpoint down the receiver's throat.
+    AgreementPointNotFound(BlockHash),
+}
+
+fn poll_once<C>(emitter: &Emitter<C>) -> Result<PollResponse, bitcoincore_rpc::Error>
+where
+    C: bitcoincore_rpc::RpcApi,
+{
+    let client = emitter.client;
+
+    if let Some(last_res) = &emitter.last_block {
+        let next_hash = if last_res.height < emitter.start_height as _ {
+            // enforce start height
+            let next_hash = client.get_block_hash(emitter.start_height as _)?;
+            // make sure last emission is still in best chain
+            if client.get_block_hash(last_res.height as _)? != last_res.hash {
+                return Ok(PollResponse::BlockNotInBestChain);
+            }
+            next_hash
+        } else {
+            match last_res.nextblockhash {
+                None => return Ok(PollResponse::NoMoreBlocks),
+                Some(next_hash) => next_hash,
+            }
+        };
+
+        let res = client.get_block_info(&next_hash)?;
+        if res.confirmations < 0 {
+            return Ok(PollResponse::BlockNotInBestChain);
+        }
+
+        return Ok(PollResponse::Block(res));
+    }
+
+    for cp in emitter.last_cp.iter() {
+        let res = match client.get_block_info(&cp.hash()) {
+            // block not in best chain
+            Ok(res) if res.confirmations < 0 => continue,
+            Ok(res) => res,
+            Err(e) if e.is_not_found_error() => {
+                if cp.height() > 0 {
+                    continue;
+                }
+                // if we can't find genesis block, we can't create an update that connects
+                break;
+            }
+            Err(e) => return Err(e),
+        };
+
+        // agreement point found
+        return Ok(PollResponse::AgreementFound(res, cp));
+    }
+
+    let genesis_hash = client.get_block_hash(0)?;
+    Ok(PollResponse::AgreementPointNotFound(genesis_hash))
+}
+
+fn poll<C, V, F>(
+    emitter: &mut Emitter<C>,
+    get_item: F,
+) -> Result<Option<(CheckPoint, V)>, bitcoincore_rpc::Error>
+where
+    C: bitcoincore_rpc::RpcApi,
+    F: Fn(&BlockHash) -> Result<V, bitcoincore_rpc::Error>,
+{
+    loop {
+        match poll_once(emitter)? {
+            PollResponse::Block(res) => {
+                let height = res.height as u32;
+                let hash = res.hash;
+                let item = get_item(&hash)?;
+
+                let new_cp = emitter
+                    .last_cp
+                    .clone()
+                    .push(BlockId { height, hash })
+                    .expect("must push");
+                emitter.last_cp = new_cp.clone();
+                emitter.last_block = Some(res);
+                return Ok(Some((new_cp, item)));
+            }
+            PollResponse::NoMoreBlocks => {
+                emitter.last_block = None;
+                return Ok(None);
+            }
+            PollResponse::BlockNotInBestChain => {
+                emitter.last_block = None;
+                continue;
+            }
+            PollResponse::AgreementFound(res, cp) => {
+                let agreement_h = res.height as u32;
+
+                // The tip during the last mempool emission needs to in the best chain, we reduce
+                // it if it is not.
+                if let Some(h) = emitter.last_mempool_tip.as_mut() {
+                    if *h > agreement_h {
+                        *h = agreement_h;
+                    }
+                }
+
+                // get rid of evicted blocks
+                emitter.last_cp = cp;
+                emitter.last_block = Some(res);
+                continue;
+            }
+            PollResponse::AgreementPointNotFound(genesis_hash) => {
+                emitter.last_cp = CheckPoint::new(BlockId {
+                    height: 0,
+                    hash: genesis_hash,
+                });
+                emitter.last_block = None;
+                continue;
+            }
+        }
+    }
+}
+
+/// Extends [`bitcoincore_rpc::Error`].
+pub trait BitcoindRpcErrorExt {
+    /// Returns whether the error is a "not found" error.
+    ///
+    /// This is useful since [`Emitter`] emits [`Result<_, bitcoincore_rpc::Error>`]s as
+    /// [`Iterator::Item`].
+    fn is_not_found_error(&self) -> bool;
+}
+
+impl BitcoindRpcErrorExt for bitcoincore_rpc::Error {
+    fn is_not_found_error(&self) -> bool {
+        if let bitcoincore_rpc::Error::JsonRpc(bitcoincore_rpc::jsonrpc::Error::Rpc(rpc_err)) = self
+        {
+            rpc_err.code == -5
+        } else {
+            false
+        }
+    }
+}

crates/bitcoind_rpc/tests/test_emitter.rs

@@ -0,0 +1,737 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use bdk_bitcoind_rpc::Emitter;
+use bdk_chain::{
+    bitcoin::{Address, Amount, Txid},
+    keychain::Balance,
+    local_chain::{CheckPoint, LocalChain},
+    Append, BlockId, IndexedTxGraph, SpkTxOutIndex,
+};
+use bdk_testenv::{anyhow, TestEnv};
+use bitcoin::{hashes::Hash, Block, OutPoint, ScriptBuf, WScriptHash};
+use bitcoincore_rpc::RpcApi;
+
+/// Ensure that blocks are emitted in order even after reorg.
+///
+/// 1. Mine 101 blocks.
+/// 2. Emit blocks from [`Emitter`] and update the [`LocalChain`].
+/// 3. Reorg highest 6 blocks.
+/// 4. Emit blocks from [`Emitter`] and re-update the [`LocalChain`].
+#[test]
+pub fn test_sync_local_chain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let network_tip = env.rpc_client().get_block_count()?;
+    let (mut local_chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
+    let mut emitter = Emitter::new(env.rpc_client(), local_chain.tip(), 0);
+
+    // Mine some blocks and return the actual block hashes.
+    // Because initializing `ElectrsD` already mines some blocks, we must include those too when
+    // returning block hashes.
+    let exp_hashes = {
+        let mut hashes = (0..=network_tip)
+            .map(|height| env.rpc_client().get_block_hash(height))
+            .collect::<Result<Vec<_>, _>>()?;
+        hashes.extend(env.mine_blocks(101 - network_tip as usize, None)?);
+        hashes
+    };
+
+    // See if the emitter outputs the right blocks.
+    println!("first sync:");
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let hash = emission.block_hash();
+        assert_eq!(
+            emission.block_hash(),
+            exp_hashes[height as usize],
+            "emitted block hash is unexpected"
+        );
+
+        assert_eq!(
+            local_chain.apply_update(emission.checkpoint,)?,
+            BTreeMap::from([(height, Some(hash))]),
+            "chain update changeset is unexpected",
+        );
+    }
+
+    assert_eq!(
+        local_chain
+            .iter_checkpoints()
+            .map(|cp| (cp.height(), cp.hash()))
+            .collect::<BTreeSet<_>>(),
+        exp_hashes
+            .iter()
+            .enumerate()
+            .map(|(i, hash)| (i as u32, *hash))
+            .collect::<BTreeSet<_>>(),
+        "final local_chain state is unexpected",
+    );
+
+    // Perform reorg.
+    let reorged_blocks = env.reorg(6)?;
+    let exp_hashes = exp_hashes
+        .iter()
+        .take(exp_hashes.len() - reorged_blocks.len())
+        .chain(&reorged_blocks)
+        .cloned()
+        .collect::<Vec<_>>();
+
+    // See if the emitter outputs the right blocks.
+    println!("after reorg:");
+    let mut exp_height = exp_hashes.len() - reorged_blocks.len();
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let hash = emission.block_hash();
+        assert_eq!(
+            height, exp_height as u32,
+            "emitted block has unexpected height"
+        );
+
+        assert_eq!(
+            hash, exp_hashes[height as usize],
+            "emitted block is unexpected"
+        );
+
+        assert_eq!(
+            local_chain.apply_update(emission.checkpoint,)?,
+            if exp_height == exp_hashes.len() - reorged_blocks.len() {
+                core::iter::once((height, Some(hash)))
+                    .chain((height + 1..exp_hashes.len() as u32).map(|h| (h, None)))
+                    .collect::<bdk_chain::local_chain::ChangeSet>()
+            } else {
+                BTreeMap::from([(height, Some(hash))])
+            },
+            "chain update changeset is unexpected",
+        );
+
+        exp_height += 1;
+    }
+
+    assert_eq!(
+        local_chain
+            .iter_checkpoints()
+            .map(|cp| (cp.height(), cp.hash()))
+            .collect::<BTreeSet<_>>(),
+        exp_hashes
+            .iter()
+            .enumerate()
+            .map(|(i, hash)| (i as u32, *hash))
+            .collect::<BTreeSet<_>>(),
+        "final local_chain state is unexpected after reorg",
+    );
+
+    Ok(())
+}
+
+/// Ensure that [`EmittedUpdate::into_tx_graph_update`] behaves appropriately for both mempool and
+/// block updates.
+///
+/// [`EmittedUpdate::into_tx_graph_update`]: bdk_bitcoind_rpc::EmittedUpdate::into_tx_graph_update
+#[test]
+fn test_into_tx_graph() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+
+    println!("getting new addresses!");
+    let addr_0 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let addr_1 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let addr_2 = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    println!("got new addresses!");
+
+    println!("mining block!");
+    env.mine_blocks(101, None)?;
+    println!("mined blocks!");
+
+    let (mut chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
+    let mut indexed_tx_graph = IndexedTxGraph::<BlockId, _>::new({
+        let mut index = SpkTxOutIndex::<usize>::default();
+        index.insert_spk(0, addr_0.script_pubkey());
+        index.insert_spk(1, addr_1.script_pubkey());
+        index.insert_spk(2, addr_2.script_pubkey());
+        index
+    });
+
+    let emitter = &mut Emitter::new(env.rpc_client(), chain.tip(), 0);
+
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        let _ = chain.apply_update(emission.checkpoint)?;
+        let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
+        assert!(indexed_additions.is_empty());
+    }
+
+    // send 3 txs to a tracked address, these txs will be in the mempool
+    let exp_txids = {
+        let mut txids = BTreeSet::new();
+        for _ in 0..3 {
+            txids.insert(env.rpc_client().send_to_address(
+                &addr_0,
+                Amount::from_sat(10_000),
+                None,
+                None,
+                None,
+                None,
+                None,
+                None,
+            )?);
+        }
+        txids
+    };
+
+    // expect that the next block should be none and we should get 3 txs from mempool
+    {
+        // next block should be `None`
+        assert!(emitter.next_block()?.is_none());
+
+        let mempool_txs = emitter.mempool()?;
+        let indexed_additions = indexed_tx_graph.batch_insert_unconfirmed(mempool_txs);
+        assert_eq!(
+            indexed_additions
+                .graph
+                .txs
+                .iter()
+                .map(|tx| tx.compute_txid())
+                .collect::<BTreeSet<Txid>>(),
+            exp_txids,
+            "changeset should have the 3 mempool transactions",
+        );
+        assert!(indexed_additions.graph.anchors.is_empty());
+    }
+
+    // mine a block that confirms the 3 txs
+    let exp_block_hash = env.mine_blocks(1, None)?[0];
+    let exp_block_height = env.rpc_client().get_block_info(&exp_block_hash)?.height as u32;
+    let exp_anchors = exp_txids
+        .iter()
+        .map({
+            let anchor = BlockId {
+                height: exp_block_height,
+                hash: exp_block_hash,
+            };
+            move |&txid| (anchor, txid)
+        })
+        .collect::<BTreeSet<_>>();
+
+    // must receive mined block which will confirm the transactions.
+    {
+        let emission = emitter.next_block()?.expect("must get mined block");
+        let height = emission.block_height();
+        let _ = chain.apply_update(emission.checkpoint)?;
+        let indexed_additions = indexed_tx_graph.apply_block_relevant(&emission.block, height);
+        assert!(indexed_additions.graph.txs.is_empty());
+        assert!(indexed_additions.graph.txouts.is_empty());
+        assert_eq!(indexed_additions.graph.anchors, exp_anchors);
+    }
+
+    Ok(())
+}
+
+/// Ensure next block emitted after reorg is at reorg height.
+///
+/// After a reorg, if the last-emitted block height is equal or greater than the reorg height, and
+/// the fallback height is equal to or lower than the reorg height, the next block/header emission
+/// should be at the reorg height.
+///
+/// TODO: If the reorg height is lower than the fallback height, how do we find a block height to
+/// emit that can connect with our receiver chain?
+#[test]
+fn ensure_block_emitted_after_reorg_is_at_reorg_height() -> anyhow::Result<()> {
+    const EMITTER_START_HEIGHT: usize = 100;
+    const CHAIN_TIP_HEIGHT: usize = 110;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        EMITTER_START_HEIGHT as _,
+    );
+
+    env.mine_blocks(CHAIN_TIP_HEIGHT, None)?;
+    while emitter.next_header()?.is_some() {}
+
+    for reorg_count in 1..=10 {
+        let replaced_blocks = env.reorg_empty_blocks(reorg_count)?;
+        let next_emission = emitter.next_header()?.expect("must emit block after reorg");
+        assert_eq!(
+            (
+                next_emission.block_height() as usize,
+                next_emission.block_hash()
+            ),
+            replaced_blocks[0],
+            "block emitted after reorg should be at the reorg height"
+        );
+        while emitter.next_header()?.is_some() {}
+    }
+
+    Ok(())
+}
+
+fn process_block(
+    recv_chain: &mut LocalChain,
+    recv_graph: &mut IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+    block: Block,
+    block_height: u32,
+) -> anyhow::Result<()> {
+    recv_chain.apply_update(CheckPoint::from_header(&block.header, block_height))?;
+    let _ = recv_graph.apply_block(block, block_height);
+    Ok(())
+}
+
+fn sync_from_emitter<C>(
+    recv_chain: &mut LocalChain,
+    recv_graph: &mut IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+    emitter: &mut Emitter<C>,
+) -> anyhow::Result<()>
+where
+    C: bitcoincore_rpc::RpcApi,
+{
+    while let Some(emission) = emitter.next_block()? {
+        let height = emission.block_height();
+        process_block(recv_chain, recv_graph, emission.block, height)?;
+    }
+    Ok(())
+}
+
+fn get_balance(
+    recv_chain: &LocalChain,
+    recv_graph: &IndexedTxGraph<BlockId, SpkTxOutIndex<()>>,
+) -> anyhow::Result<Balance> {
+    let chain_tip = recv_chain.tip().block_id();
+    let outpoints = recv_graph.index.outpoints().clone();
+    let balance = recv_graph
+        .graph()
+        .balance(recv_chain, chain_tip, outpoints, |_, _| true);
+    Ok(balance)
+}
+
+/// If a block is reorged out, ensure that containing transactions that do not exist in the
+/// replacement block(s) become unconfirmed.
+#[test]
+fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+    const ADDITIONAL_COUNT: usize = 11;
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // setup addresses
+    let addr_to_mine = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bitcoin::Network::Regtest)?;
+
+    // setup receiver
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.rpc_client().get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<BlockId, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // mine and sync receiver up to tip
+    env.mine_blocks(PREMINE_COUNT, Some(addr_to_mine))?;
+
+    // create transactions that are tracked by our receiver
+    for _ in 0..ADDITIONAL_COUNT {
+        let txid = env.send(&addr_to_track, SEND_AMOUNT)?;
+
+        // lock outputs that send to `addr_to_track`
+        let outpoints_to_lock = env
+            .rpc_client()
+            .get_transaction(&txid, None)?
+            .transaction()?
+            .output
+            .into_iter()
+            .enumerate()
+            .filter(|(_, txo)| txo.script_pubkey == spk_to_track)
+            .map(|(vout, _)| OutPoint::new(txid, vout as _))
+            .collect::<Vec<_>>();
+        env.rpc_client().lock_unspent(&outpoints_to_lock)?;
+
+        let _ = env.mine_blocks(1, None)?;
+    }
+
+    // get emitter up to tip
+    sync_from_emitter(&mut recv_chain, &mut recv_graph, &mut emitter)?;
+
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT * ADDITIONAL_COUNT as u64,
+            ..Balance::default()
+        },
+        "initial balance must be correct",
+    );
+
+    // perform reorgs with different depths
+    for reorg_count in 1..=ADDITIONAL_COUNT {
+        env.reorg_empty_blocks(reorg_count)?;
+        sync_from_emitter(&mut recv_chain, &mut recv_graph, &mut emitter)?;
+
+        assert_eq!(
+            get_balance(&recv_chain, &recv_graph)?,
+            Balance {
+                confirmed: SEND_AMOUNT * (ADDITIONAL_COUNT - reorg_count) as u64,
+                trusted_pending: SEND_AMOUNT * reorg_count as u64,
+                ..Balance::default()
+            },
+            "reorg_count: {}",
+            reorg_count,
+        );
+    }
+
+    Ok(())
+}
+
+/// Ensure avoid-re-emission-logic is sound when [`Emitter`] is synced to tip.
+///
+/// The receiver (bdk_chain structures) is synced to the chain tip, and there is txs in the mempool.
+/// When we call Emitter::mempool multiple times, mempool txs should not be re-emitted, even if the
+/// chain tip is extended.
+#[test]
+fn mempool_avoids_re_emission() -> anyhow::Result<()> {
+    const BLOCKS_TO_MINE: usize = 101;
+    const MEMPOOL_TX_COUNT: usize = 2;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks and sync up emitter
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    env.mine_blocks(BLOCKS_TO_MINE, Some(addr.clone()))?;
+    while emitter.next_header()?.is_some() {}
+
+    // have some random txs in mempool
+    let exp_txids = (0..MEMPOOL_TX_COUNT)
+        .map(|_| env.send(&addr, Amount::from_sat(2100)))
+        .collect::<Result<BTreeSet<Txid>, _>>()?;
+
+    // the first emission should include all transactions
+    let emitted_txids = emitter
+        .mempool()?
+        .into_iter()
+        .map(|(tx, _)| tx.compute_txid())
+        .collect::<BTreeSet<Txid>>();
+    assert_eq!(
+        emitted_txids, exp_txids,
+        "all mempool txs should be emitted"
+    );
+
+    // second emission should be empty
+    assert!(
+        emitter.mempool()?.is_empty(),
+        "second emission should be empty"
+    );
+
+    // mine empty blocks + sync up our emitter -> we should still not re-emit
+    for _ in 0..BLOCKS_TO_MINE {
+        env.mine_empty_block()?;
+    }
+    while emitter.next_header()?.is_some() {}
+    assert!(
+        emitter.mempool()?.is_empty(),
+        "third emission, after chain tip is extended, should also be empty"
+    );
+
+    Ok(())
+}
+
+/// Ensure mempool tx is still re-emitted if [`Emitter`] has not reached the tx's introduction
+/// height.
+///
+/// We introduce a mempool tx after each block, where blocks are empty (does not confirm previous
+/// mempool txs). Then we emit blocks from [`Emitter`] (intertwining `mempool` calls). We check
+/// that `mempool` should always re-emit txs that have introduced at a height greater than the last
+/// emitted block height.
+#[test]
+fn mempool_re_emits_if_tx_introduction_height_not_reached() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+    const MEMPOOL_TX_COUNT: usize = 21;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks to get initial balance, sync emitter up to tip
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
+    while emitter.next_header()?.is_some() {}
+
+    // mine blocks to introduce txs to mempool at different heights
+    let tx_introductions = (0..MEMPOOL_TX_COUNT)
+        .map(|_| -> anyhow::Result<_> {
+            let (height, _) = env.mine_empty_block()?;
+            let txid = env.send(&addr, Amount::from_sat(2100))?;
+            Ok((height, txid))
+        })
+        .collect::<anyhow::Result<BTreeSet<_>>>()?;
+
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.compute_txid())
+            .collect::<BTreeSet<_>>(),
+        tx_introductions.iter().map(|&(_, txid)| txid).collect(),
+        "first mempool emission should include all txs",
+    );
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.compute_txid())
+            .collect::<BTreeSet<_>>(),
+        tx_introductions.iter().map(|&(_, txid)| txid).collect(),
+        "second mempool emission should still include all txs",
+    );
+
+    // At this point, the emitter has seen all mempool transactions. It should only re-emit those
+    // that have introduction heights less than the emitter's last-emitted block tip.
+    while let Some(emission) = emitter.next_header()? {
+        let height = emission.block_height();
+        // We call `mempool()` twice.
+        // The second call (at height `h`) should skip the tx introduced at height `h`.
+        for try_index in 0..2 {
+            let exp_txids = tx_introductions
+                .range((height as usize + try_index, Txid::all_zeros())..)
+                .map(|&(_, txid)| txid)
+                .collect::<BTreeSet<_>>();
+            let emitted_txids = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.compute_txid())
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                emitted_txids, exp_txids,
+                "\n emission {} (try {}) must only contain txs introduced at that height or lower: \n\t missing: {:?} \n\t extra: {:?}",
+                height,
+                try_index,
+                exp_txids
+                    .difference(&emitted_txids)
+                    .map(|txid| (txid, tx_introductions.iter().find_map(|(h, id)| if id == txid { Some(h) } else { None }).unwrap()))
+                    .collect::<Vec<_>>(),
+                emitted_txids
+                    .difference(&exp_txids)
+                    .map(|txid| (txid, tx_introductions.iter().find_map(|(h, id)| if id == txid { Some(h) } else { None }).unwrap()))
+                    .collect::<Vec<_>>(),
+            );
+        }
+    }
+
+    Ok(())
+}
+
+/// Ensure we force re-emit all mempool txs after reorg.
+#[test]
+fn mempool_during_reorg() -> anyhow::Result<()> {
+    const TIP_DIFF: usize = 10;
+    const PREMINE_COUNT: usize = 101;
+
+    let env = TestEnv::new()?;
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        0,
+    );
+
+    // mine blocks to get initial balance
+    let addr = env
+        .rpc_client()
+        .get_new_address(None, None)?
+        .assume_checked();
+    env.mine_blocks(PREMINE_COUNT, Some(addr.clone()))?;
+
+    // introduce mempool tx at each block extension
+    for _ in 0..TIP_DIFF {
+        env.mine_empty_block()?;
+        env.send(&addr, Amount::from_sat(2100))?;
+    }
+
+    // sync emitter to tip, first mempool emission should include all txs (as we haven't emitted
+    // from the mempool yet)
+    while emitter.next_header()?.is_some() {}
+    assert_eq!(
+        emitter
+            .mempool()?
+            .into_iter()
+            .map(|(tx, _)| tx.compute_txid())
+            .collect::<BTreeSet<_>>(),
+        env.rpc_client()
+            .get_raw_mempool()?
+            .into_iter()
+            .collect::<BTreeSet<_>>(),
+        "first mempool emission should include all txs",
+    );
+
+    // perform reorgs at different heights, these reorgs will not confirm transactions in the
+    // mempool
+    for reorg_count in 1..TIP_DIFF {
+        println!("REORG COUNT: {}", reorg_count);
+        env.reorg_empty_blocks(reorg_count)?;
+
+        // This is a map of mempool txids to tip height where the tx was introduced to the mempool
+        // we recalculate this at every loop as reorgs may evict transactions from mempool. We use
+        // the introduction height to determine whether we expect a tx to appear in a mempool
+        // emission.
+        // TODO: How can have have reorg logic in `TestEnv` NOT blacklast old blocks first?
+        let tx_introductions = dbg!(env
+            .rpc_client()
+            .get_raw_mempool_verbose()?
+            .into_iter()
+            .map(|(txid, entry)| (txid, entry.height as usize))
+            .collect::<BTreeMap<_, _>>());
+
+        // `next_header` emits the replacement block of the reorg
+        if let Some(emission) = emitter.next_header()? {
+            let height = emission.block_height();
+            println!("\t- replacement height: {}", height);
+
+            // the mempool emission (that follows the first block emission after reorg) should only
+            // include mempool txs introduced at reorg height or greater
+            let mempool = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.compute_txid())
+                .collect::<BTreeSet<_>>();
+            let exp_mempool = tx_introductions
+                .iter()
+                .filter(|(_, &intro_h)| intro_h >= (height as usize))
+                .map(|(&txid, _)| txid)
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                mempool, exp_mempool,
+                "the first mempool emission after reorg should only include mempool txs introduced at reorg height or greater"
+            );
+
+            let mempool = emitter
+                .mempool()?
+                .into_iter()
+                .map(|(tx, _)| tx.compute_txid())
+                .collect::<BTreeSet<_>>();
+            let exp_mempool = tx_introductions
+                .iter()
+                .filter(|&(_, &intro_height)| intro_height > (height as usize))
+                .map(|(&txid, _)| txid)
+                .collect::<BTreeSet<_>>();
+            assert_eq!(
+                mempool, exp_mempool,
+                "following mempool emissions after reorg should exclude mempool introduction heights <= last emitted block height: \n\t missing: {:?} \n\t extra: {:?}",
+                exp_mempool
+                    .difference(&mempool)
+                    .map(|txid| (txid, tx_introductions.get(txid).unwrap()))
+                    .collect::<Vec<_>>(),
+                mempool
+                    .difference(&exp_mempool)
+                    .map(|txid| (txid, tx_introductions.get(txid).unwrap()))
+                    .collect::<Vec<_>>(),
+            );
+        }
+
+        // sync emitter to tip
+        while emitter.next_header()?.is_some() {}
+    }
+
+    Ok(())
+}
+
+/// If blockchain re-org includes the start height, emit new start height block
+///
+/// 1. mine 101 blocks
+/// 2. emit blocks 99a, 100a
+/// 3. invalidate blocks 99a, 100a, 101a
+/// 4. mine new blocks 99b, 100b, 101b
+/// 5. emit block 99b
+///
+/// The block hash of 99b should be different than 99a, but their previous block hashes should
+/// be the same.
+#[test]
+fn no_agreement_point() -> anyhow::Result<()> {
+    const PREMINE_COUNT: usize = 101;
+
+    let env = TestEnv::new()?;
+
+    // start height is 99
+    let mut emitter = Emitter::new(
+        env.rpc_client(),
+        CheckPoint::new(BlockId {
+            height: 0,
+            hash: env.rpc_client().get_block_hash(0)?,
+        }),
+        (PREMINE_COUNT - 2) as u32,
+    );
+
+    // mine 101 blocks
+    env.mine_blocks(PREMINE_COUNT, None)?;
+
+    // emit block 99a
+    let block_header_99a = emitter.next_header()?.expect("block 99a header").block;
+    let block_hash_99a = block_header_99a.block_hash();
+    let block_hash_98a = block_header_99a.prev_blockhash;
+
+    // emit block 100a
+    let block_header_100a = emitter.next_header()?.expect("block 100a header").block;
+    let block_hash_100a = block_header_100a.block_hash();
+
+    // get hash for block 101a
+    let block_hash_101a = env.rpc_client().get_block_hash(101)?;
+
+    // invalidate blocks 99a, 100a, 101a
+    env.rpc_client().invalidate_block(&block_hash_99a)?;
+    env.rpc_client().invalidate_block(&block_hash_100a)?;
+    env.rpc_client().invalidate_block(&block_hash_101a)?;
+
+    // mine new blocks 99b, 100b, 101b
+    env.mine_blocks(3, None)?;
+
+    // emit block header 99b
+    let block_header_99b = emitter.next_header()?.expect("block 99b header").block;
+    let block_hash_99b = block_header_99b.block_hash();
+    let block_hash_98b = block_header_99b.prev_blockhash;
+
+    assert_ne!(block_hash_99a, block_hash_99b);
+    assert_eq!(block_hash_98a, block_hash_98b);
+
+    Ok(())
+}

crates/chain/Cargo.toml

@@ -1,8 +1,8 @@
 [package]
 name = "bdk_chain"
-version = "0.5.0"
+version = "0.16.0"
 edition = "2021"
-rust-version = "1.57"
+rust-version = "1.63"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk_chain"
@@ -13,19 +13,18 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-# For no-std, remember to enable the bitcoin/no-std feature
-bitcoin = { version = "0.29", default-features = false }
-serde_crate = { package = "serde", version = "1", optional = true, features = ["derive"] }
+bitcoin = { version = "0.32.0", default-features = false }
+serde_crate = { package = "serde", version = "1", optional = true, features = ["derive", "rc"] }
 
 # Use hashbrown as a feature flag to have HashSet and HashMap from it.
-# note version 0.13 breaks outs MSRV.
-hashbrown = { version = "0.11", optional = true, features = ["serde"] }
-miniscript = { version = "9.0.0", optional = true, default-features = false }
+hashbrown = { version = "0.9.1", optional = true, features = ["serde"] }
+miniscript = { version = "12.0.0", optional = true, default-features = false }
 
 [dev-dependencies]
 rand = "0.8"
+proptest = "1.2.0"
 
 [features]
-default = ["std"]
-std = ["bitcoin/std", "miniscript/std"]
-serde = ["serde_crate", "bitcoin/serde"]
+default = ["std", "miniscript"]
+std = ["bitcoin/std", "miniscript?/std"]
+serde = ["serde_crate", "bitcoin/serde", "miniscript?/serde"]

crates/chain/src/chain_data.rs

@@ -1,6 +1,6 @@
 use bitcoin::{hashes::Hash, BlockHash, OutPoint, TxOut, Txid};
 
-use crate::{Anchor, COINBASE_MATURITY};
+use crate::{Anchor, AnchorFromBlockPosition, COINBASE_MATURITY};
 
 /// Represents the observed position of some chain data.
 ///
@@ -9,7 +9,7 @@ use crate::{Anchor, COINBASE_MATURITY};
 pub enum ChainPosition<A> {
     /// The chain data is seen as confirmed, and in anchored by `A`.
     Confirmed(A),
-    /// The chain data is seen in mempool at this given timestamp.
+    /// The chain data is not confirmed and last seen in the mempool at this timestamp.
     Unconfirmed(u64),
 }
 
@@ -48,14 +48,14 @@ impl<A: Anchor> ChainPosition<A> {
     serde(crate = "serde_crate")
 )]
 pub enum ConfirmationTime {
-    /// The confirmed variant.
+    /// The transaction is confirmed
     Confirmed {
         /// Confirmation height.
         height: u32,
         /// Confirmation time in unix seconds.
         time: u64,
     },
-    /// The unconfirmed variant.
+    /// The transaction is unconfirmed
     Unconfirmed {
         /// The last-seen timestamp in unix seconds.
         last_seen: u64,
@@ -74,19 +74,22 @@ impl ConfirmationTime {
     }
 }
 
-impl From<ChainPosition<ConfirmationTimeAnchor>> for ConfirmationTime {
-    fn from(observed_as: ChainPosition<ConfirmationTimeAnchor>) -> Self {
+impl From<ChainPosition<ConfirmationTimeHeightAnchor>> for ConfirmationTime {
+    fn from(observed_as: ChainPosition<ConfirmationTimeHeightAnchor>) -> Self {
         match observed_as {
             ChainPosition::Confirmed(a) => Self::Confirmed {
                 height: a.confirmation_height,
                 time: a.confirmation_time,
             },
-            ChainPosition::Unconfirmed(_) => Self::Unconfirmed { last_seen: 0 },
+            ChainPosition::Unconfirmed(last_seen) => Self::Unconfirmed { last_seen },
         }
     }
 }
 
 /// A reference to a block in the canonical chain.
+///
+/// `BlockId` implements [`Anchor`]. When a transaction is anchored to `BlockId`, the confirmation
+/// block and anchor block are the same block.
 #[derive(Debug, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
@@ -100,11 +103,23 @@ pub struct BlockId {
     pub hash: BlockHash,
 }
 
+impl Anchor for BlockId {
+    fn anchor_block(&self) -> Self {
+        *self
+    }
+}
+
+impl AnchorFromBlockPosition for BlockId {
+    fn from_block_position(_block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        block_id
+    }
+}
+
 impl Default for BlockId {
     fn default() -> Self {
         Self {
             height: Default::default(),
-            hash: BlockHash::from_inner([0u8; 32]),
+            hash: BlockHash::all_zeros(),
         }
     }
 }
@@ -131,6 +146,10 @@ impl From<(&u32, &BlockHash)> for BlockId {
 }
 
 /// An [`Anchor`] implementation that also records the exact confirmation height of the transaction.
+///
+/// Note that the confirmation block and the anchor block can be different here.
+///
+/// Refer to [`Anchor`] for more details.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
@@ -138,13 +157,12 @@ impl From<(&u32, &BlockHash)> for BlockId {
     serde(crate = "serde_crate")
 )]
 pub struct ConfirmationHeightAnchor {
-    /// The anchor block.
-    pub anchor_block: BlockId,
-
     /// The exact confirmation height of the transaction.
     ///
     /// It is assumed that this value is never larger than the height of the anchor block.
     pub confirmation_height: u32,
+    /// The anchor block.
+    pub anchor_block: BlockId,
 }
 
 impl Anchor for ConfirmationHeightAnchor {
@@ -157,24 +175,37 @@ impl Anchor for ConfirmationHeightAnchor {
     }
 }
 
+impl AnchorFromBlockPosition for ConfirmationHeightAnchor {
+    fn from_block_position(_block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        Self {
+            anchor_block: block_id,
+            confirmation_height: block_id.height,
+        }
+    }
+}
+
 /// An [`Anchor`] implementation that also records the exact confirmation time and height of the
 /// transaction.
+///
+/// Note that the confirmation block and the anchor block can be different here.
+///
+/// Refer to [`Anchor`] for more details.
 #[derive(Debug, Default, Clone, PartialEq, Eq, Copy, PartialOrd, Ord, core::hash::Hash)]
 #[cfg_attr(
     feature = "serde",
     derive(serde::Deserialize, serde::Serialize),
     serde(crate = "serde_crate")
 )]
-pub struct ConfirmationTimeAnchor {
+pub struct ConfirmationTimeHeightAnchor {
+    /// The confirmation height of the transaction being anchored.
+    pub confirmation_height: u32,
+    /// The confirmation time of the transaction being anchored.
+    pub confirmation_time: u64,
     /// The anchor block.
     pub anchor_block: BlockId,
-    /// The confirmation height of the chain data being anchored.
-    pub confirmation_height: u32,
-    /// The confirmation time of the chain data being anchored.
-    pub confirmation_time: u64,
 }
 
-impl Anchor for ConfirmationTimeAnchor {
+impl Anchor for ConfirmationTimeHeightAnchor {
     fn anchor_block(&self) -> BlockId {
         self.anchor_block
     }
@@ -183,15 +214,26 @@ impl Anchor for ConfirmationTimeAnchor {
         self.confirmation_height
     }
 }
+
+impl AnchorFromBlockPosition for ConfirmationTimeHeightAnchor {
+    fn from_block_position(block: &bitcoin::Block, block_id: BlockId, _tx_pos: usize) -> Self {
+        Self {
+            anchor_block: block_id,
+            confirmation_height: block_id.height,
+            confirmation_time: block.header.time as _,
+        }
+    }
+}
+
 /// A `TxOut` with as much data as we can retrieve about it
 #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)]
 pub struct FullTxOut<A> {
+    /// The position of the transaction in `outpoint` in the overall chain.
+    pub chain_position: ChainPosition<A>,
     /// The location of the `TxOut`.
     pub outpoint: OutPoint,
     /// The `TxOut`.
     pub txout: TxOut,
-    /// The position of the transaction in `outpoint` in the overall chain.
-    pub chain_position: ChainPosition<A>,
     /// The txid and chain position of the transaction (if any) that has spent this output.
     pub spent_by: Option<(ChainPosition<A>, Txid)>,
     /// Whether this output is on a coinbase transaction.
@@ -202,7 +244,7 @@ impl<A: Anchor> FullTxOut<A> {
     /// Whether the `txout` is considered mature.
     ///
     /// Depending on the implementation of [`confirmation_height_upper_bound`] in [`Anchor`], this
-    /// method may return false-negatives. In other words, interpretted confirmation count may be
+    /// method may return false-negatives. In other words, interpreted confirmation count may be
     /// less than the actual value.
     ///
     /// [`confirmation_height_upper_bound`]: Anchor::confirmation_height_upper_bound
@@ -226,10 +268,10 @@ impl<A: Anchor> FullTxOut<A> {
 
     /// Whether the utxo is/was/will be spendable with chain `tip`.
     ///
-    /// This method does not take into account the locktime.
+    /// This method does not take into account the lock time.
     ///
     /// Depending on the implementation of [`confirmation_height_upper_bound`] in [`Anchor`], this
-    /// method may return false-negatives. In other words, interpretted confirmation count may be
+    /// method may return false-negatives. In other words, interpreted confirmation count may be
     /// less than the actual value.
     ///
     /// [`confirmation_height_upper_bound`]: Anchor::confirmation_height_upper_bound
@@ -256,3 +298,35 @@ impl<A: Anchor> FullTxOut<A> {
         true
     }
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn chain_position_ord() {
+        let unconf1 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(10);
+        let unconf2 = ChainPosition::<ConfirmationHeightAnchor>::Unconfirmed(20);
+        let conf1 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
+            confirmation_height: 9,
+            anchor_block: BlockId {
+                height: 20,
+                ..Default::default()
+            },
+        });
+        let conf2 = ChainPosition::Confirmed(ConfirmationHeightAnchor {
+            confirmation_height: 12,
+            anchor_block: BlockId {
+                height: 15,
+                ..Default::default()
+            },
+        });
+
+        assert!(unconf2 > unconf1, "higher last_seen means higher ord");
+        assert!(unconf1 > conf1, "unconfirmed is higher ord than confirmed");
+        assert!(
+            conf2 > conf1,
+            "confirmation_height is higher then it should be higher ord"
+        );
+    }
+}

crates/chain/src/chain_oracle.rs

@@ -3,7 +3,7 @@ use crate::BlockId;
 /// Represents a service that tracks the blockchain.
 ///
 /// The main method is [`is_block_in_chain`] which determines whether a given block of [`BlockId`]
-/// is an ancestor of another "static block".
+/// is an ancestor of the `chain_tip`.
 ///
 /// [`is_block_in_chain`]: Self::is_block_in_chain
 pub trait ChainOracle {
@@ -21,5 +21,5 @@ pub trait ChainOracle {
     ) -> Result<Option<bool>, Self::Error>;
 
     /// Get the best chain's chain tip.
-    fn get_chain_tip(&self) -> Result<Option<BlockId>, Self::Error>;
+    fn get_chain_tip(&self) -> Result<BlockId, Self::Error>;
 }

crates/chain/src/changeset.rs

@@ -0,0 +1,89 @@
+/// A changeset containing [`crate`] structures typically persisted together.
+#[cfg(feature = "miniscript")]
+#[derive(Debug, Clone, PartialEq)]
+#[cfg_attr(
+    feature = "serde",
+    derive(crate::serde::Deserialize, crate::serde::Serialize),
+    serde(
+        crate = "crate::serde",
+        bound(
+            deserialize = "A: Ord + crate::serde::Deserialize<'de>, K: Ord + crate::serde::Deserialize<'de>",
+            serialize = "A: Ord + crate::serde::Serialize, K: Ord + crate::serde::Serialize",
+        ),
+    )
+)]
+pub struct CombinedChangeSet<K, A> {
+    /// Changes to the [`LocalChain`](crate::local_chain::LocalChain).
+    pub chain: crate::local_chain::ChangeSet,
+    /// Changes to [`IndexedTxGraph`](crate::indexed_tx_graph::IndexedTxGraph).
+    pub indexed_tx_graph: crate::indexed_tx_graph::ChangeSet<A, crate::keychain::ChangeSet<K>>,
+    /// Stores the network type of the transaction data.
+    pub network: Option<bitcoin::Network>,
+}
+
+#[cfg(feature = "miniscript")]
+impl<K, A> core::default::Default for CombinedChangeSet<K, A> {
+    fn default() -> Self {
+        Self {
+            chain: core::default::Default::default(),
+            indexed_tx_graph: core::default::Default::default(),
+            network: None,
+        }
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl<K: Ord, A: crate::Anchor> crate::Append for CombinedChangeSet<K, A> {
+    fn append(&mut self, other: Self) {
+        crate::Append::append(&mut self.chain, other.chain);
+        crate::Append::append(&mut self.indexed_tx_graph, other.indexed_tx_graph);
+        if other.network.is_some() {
+            debug_assert!(
+                self.network.is_none() || self.network == other.network,
+                "network type must either be just introduced or remain the same"
+            );
+            self.network = other.network;
+        }
+    }
+
+    fn is_empty(&self) -> bool {
+        self.chain.is_empty() && self.indexed_tx_graph.is_empty() && self.network.is_none()
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl<K, A> From<crate::local_chain::ChangeSet> for CombinedChangeSet<K, A> {
+    fn from(chain: crate::local_chain::ChangeSet) -> Self {
+        Self {
+            chain,
+            ..Default::default()
+        }
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl<K, A> From<crate::indexed_tx_graph::ChangeSet<A, crate::keychain::ChangeSet<K>>>
+    for CombinedChangeSet<K, A>
+{
+    fn from(
+        indexed_tx_graph: crate::indexed_tx_graph::ChangeSet<A, crate::keychain::ChangeSet<K>>,
+    ) -> Self {
+        Self {
+            indexed_tx_graph,
+            ..Default::default()
+        }
+    }
+}
+
+#[cfg(feature = "miniscript")]
+impl<K, A> From<crate::keychain::ChangeSet<K>> for CombinedChangeSet<K, A> {
+    fn from(indexer: crate::keychain::ChangeSet<K>) -> Self {
+        Self {
+            indexed_tx_graph: crate::indexed_tx_graph::ChangeSet {
+                indexer,
+                ..Default::default()
+            },
+            ..Default::default()
+        }
+    }
+}

crates/chain/src/descriptor_ext.rs

@@ -1,16 +1,44 @@
-use crate::miniscript::{Descriptor, DescriptorPublicKey};
+use crate::{
+    alloc::{string::ToString, vec::Vec},
+    miniscript::{Descriptor, DescriptorPublicKey},
+};
+use bitcoin::hashes::{hash_newtype, sha256, Hash};
+
+hash_newtype! {
+    /// Represents the ID of a descriptor, defined as the sha256 hash of
+    /// the descriptor string, checksum excluded.
+    ///
+    /// This is useful for having a fixed-length unique representation of a descriptor,
+    /// in particular, we use it to persist application state changes related to the
+    /// descriptor without having to re-write the whole descriptor each time.
+    ///
+    pub struct DescriptorId(pub sha256::Hash);
+}
 
 /// A trait to extend the functionality of a miniscript descriptor.
 pub trait DescriptorExt {
     /// Returns the minimum value (in satoshis) at which an output is broadcastable.
+    /// Panics if the descriptor wildcard is hardened.
     fn dust_value(&self) -> u64;
+
+    /// Returns the descriptor id, calculated as the sha256 of the descriptor, checksum not
+    /// included.
+    fn descriptor_id(&self) -> DescriptorId;
 }
 
 impl DescriptorExt for Descriptor<DescriptorPublicKey> {
     fn dust_value(&self) -> u64 {
         self.at_derivation_index(0)
+            .expect("descriptor can't have hardened derivation")
             .script_pubkey()
-            .dust_value()
+            .minimal_non_dust()
             .to_sat()
     }
+
+    fn descriptor_id(&self) -> DescriptorId {
+        let desc = self.to_string();
+        let desc_without_checksum = desc.split('#').next().expect("Must be here");
+        let descriptor_bytes = <Vec<u8>>::from(desc_without_checksum.as_bytes());
+        DescriptorId(sha256::Hash::hash(&descriptor_bytes))
+    }
 }

crates/chain/src/indexed_tx_graph.rs

@@ -1,19 +1,16 @@
-//! Contains the [`IndexedTxGraph`] structure and associated types.
-//!
-//! This is essentially a [`TxGraph`] combined with an indexer.
-
+//! Contains the [`IndexedTxGraph`] and associated types. Refer to the
+//! [`IndexedTxGraph`] documentation for more.
 use alloc::vec::Vec;
-use bitcoin::{OutPoint, Transaction, TxOut};
+use bitcoin::{Block, OutPoint, Transaction, TxOut, Txid};
 
 use crate::{
-    keychain::DerivationAdditions,
-    tx_graph::{Additions, TxGraph},
-    Anchor, Append,
+    tx_graph::{self, TxGraph},
+    Anchor, AnchorFromBlockPosition, Append, BlockId,
 };
 
-/// A struct that combines [`TxGraph`] and an [`Indexer`] implementation.
+/// The [`IndexedTxGraph`] combines a [`TxGraph`] and an [`Indexer`] implementation.
 ///
-/// This structure ensures that [`TxGraph`] and [`Indexer`] are updated atomically.
+/// It ensures that [`TxGraph`] and [`Indexer`] are updated atomically.
 #[derive(Debug)]
 pub struct IndexedTxGraph<A, I> {
     /// Transaction index.
@@ -46,128 +43,233 @@ impl<A, I> IndexedTxGraph<A, I> {
 }
 
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I> {
-    /// Applies the [`IndexedAdditions`] to the [`IndexedTxGraph`].
-    pub fn apply_additions(&mut self, additions: IndexedAdditions<A, I::Additions>) {
-        let IndexedAdditions {
-            graph_additions,
-            index_additions,
-        } = additions;
+    /// Applies the [`ChangeSet`] to the [`IndexedTxGraph`].
+    pub fn apply_changeset(&mut self, changeset: ChangeSet<A, I::ChangeSet>) {
+        self.index.apply_changeset(changeset.indexer);
 
-        self.index.apply_additions(index_additions);
-
-        for tx in &graph_additions.txs {
+        for tx in &changeset.graph.txs {
             self.index.index_tx(tx);
         }
-        for (&outpoint, txout) in &graph_additions.txouts {
+        for (&outpoint, txout) in &changeset.graph.txouts {
             self.index.index_txout(outpoint, txout);
         }
 
-        self.graph.apply_additions(graph_additions);
+        self.graph.apply_changeset(changeset.graph);
+    }
+
+    /// Determines the [`ChangeSet`] between `self` and an empty [`IndexedTxGraph`].
+    pub fn initial_changeset(&self) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.initial_changeset();
+        let indexer = self.index.initial_changeset();
+        ChangeSet { graph, indexer }
     }
 }
 
 impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
 where
-    I::Additions: Default + Append,
+    I::ChangeSet: Default + Append,
 {
+    fn index_tx_graph_changeset(
+        &mut self,
+        tx_graph_changeset: &tx_graph::ChangeSet<A>,
+    ) -> I::ChangeSet {
+        let mut changeset = I::ChangeSet::default();
+        for added_tx in &tx_graph_changeset.txs {
+            changeset.append(self.index.index_tx(added_tx));
+        }
+        for (&added_outpoint, added_txout) in &tx_graph_changeset.txouts {
+            changeset.append(self.index.index_txout(added_outpoint, added_txout));
+        }
+        changeset
+    }
+
     /// Apply an `update` directly.
     ///
-    /// `update` is a [`TxGraph<A>`] and the resultant changes is returned as [`IndexedAdditions`].
-    pub fn apply_update(&mut self, update: TxGraph<A>) -> IndexedAdditions<A, I::Additions> {
-        let graph_additions = self.graph.apply_update(update);
-
-        let mut index_additions = I::Additions::default();
-        for added_tx in &graph_additions.txs {
-            index_additions.append(self.index.index_tx(added_tx));
-        }
-        for (&added_outpoint, added_txout) in &graph_additions.txouts {
-            index_additions.append(self.index.index_txout(added_outpoint, added_txout));
-        }
-
-        IndexedAdditions {
-            graph_additions,
-            index_additions,
-        }
+    /// `update` is a [`TxGraph<A>`] and the resultant changes is returned as [`ChangeSet`].
+    pub fn apply_update(&mut self, update: TxGraph<A>) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.apply_update(update);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
     /// Insert a floating `txout` of given `outpoint`.
-    pub fn insert_txout(
-        &mut self,
-        outpoint: OutPoint,
-        txout: &TxOut,
-    ) -> IndexedAdditions<A, I::Additions> {
-        let mut update = TxGraph::<A>::default();
-        let _ = update.insert_txout(outpoint, txout.clone());
-        self.apply_update(update)
+    pub fn insert_txout(&mut self, outpoint: OutPoint, txout: TxOut) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.insert_txout(outpoint, txout);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
     /// Insert and index a transaction into the graph.
-    ///
-    /// `anchors` can be provided to anchor the transaction to various blocks. `seen_at` is a
-    /// unix timestamp of when the transaction is last seen.
-    pub fn insert_tx(
-        &mut self,
-        tx: &Transaction,
-        anchors: impl IntoIterator<Item = A>,
-        seen_at: Option<u64>,
-    ) -> IndexedAdditions<A, I::Additions> {
-        let txid = tx.txid();
-
-        let mut update = TxGraph::<A>::default();
-        if self.graph.get_tx(txid).is_none() {
-            let _ = update.insert_tx(tx.clone());
-        }
-        for anchor in anchors.into_iter() {
-            let _ = update.insert_anchor(txid, anchor);
-        }
-        if let Some(seen_at) = seen_at {
-            let _ = update.insert_seen_at(txid, seen_at);
-        }
-
-        self.apply_update(update)
+    pub fn insert_tx(&mut self, tx: Transaction) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.insert_tx(tx);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 
-    /// Insert relevant transactions from the given `txs` iterator.
+    /// Insert an `anchor` for a given transaction.
+    pub fn insert_anchor(&mut self, txid: Txid, anchor: A) -> ChangeSet<A, I::ChangeSet> {
+        self.graph.insert_anchor(txid, anchor).into()
+    }
+
+    /// Insert a unix timestamp of when a transaction is seen in the mempool.
+    ///
+    /// This is used for transaction conflict resolution in [`TxGraph`] where the transaction with
+    /// the later last-seen is prioritized.
+    pub fn insert_seen_at(&mut self, txid: Txid, seen_at: u64) -> ChangeSet<A, I::ChangeSet> {
+        self.graph.insert_seen_at(txid, seen_at).into()
+    }
+
+    /// Batch insert transactions, filtering out those that are irrelevant.
     ///
     /// Relevancy is determined by the [`Indexer::is_tx_relevant`] implementation of `I`. Irrelevant
     /// transactions in `txs` will be ignored. `txs` do not need to be in topological order.
-    ///
-    /// `anchors` can be provided to anchor the transactions to blocks. `seen_at` is a unix
-    /// timestamp of when the transactions are last seen.
-    pub fn insert_relevant_txs<'t>(
+    pub fn batch_insert_relevant<'t>(
         &mut self,
         txs: impl IntoIterator<Item = (&'t Transaction, impl IntoIterator<Item = A>)>,
-        seen_at: Option<u64>,
-    ) -> IndexedAdditions<A, I::Additions> {
+    ) -> ChangeSet<A, I::ChangeSet> {
         // The algorithm below allows for non-topologically ordered transactions by using two loops.
         // This is achieved by:
         // 1. insert all txs into the index. If they are irrelevant then that's fine it will just
         //    not store anything about them.
         // 2. decide whether to insert them into the graph depending on whether `is_tx_relevant`
         //    returns true or not. (in a second loop).
-        let mut additions = IndexedAdditions::<A, I::Additions>::default();
-        let mut transactions = Vec::new();
-        for (tx, anchors) in txs.into_iter() {
-            additions.index_additions.append(self.index.index_tx(tx));
-            transactions.push((tx, anchors));
+        let txs = txs.into_iter().collect::<Vec<_>>();
+
+        let mut indexer = I::ChangeSet::default();
+        for (tx, _) in &txs {
+            indexer.append(self.index.index_tx(tx));
         }
-        additions.append(
-            transactions
-                .into_iter()
-                .filter_map(|(tx, anchors)| match self.index.is_tx_relevant(tx) {
-                    true => Some(self.insert_tx(tx, anchors, seen_at)),
-                    false => None,
-                })
-                .fold(Default::default(), |mut acc, other| {
-                    acc.append(other);
-                    acc
-                }),
+
+        let mut graph = tx_graph::ChangeSet::default();
+        for (tx, anchors) in txs {
+            if self.index.is_tx_relevant(tx) {
+                let txid = tx.compute_txid();
+                graph.append(self.graph.insert_tx(tx.clone()));
+                for anchor in anchors {
+                    graph.append(self.graph.insert_anchor(txid, anchor));
+                }
+            }
+        }
+
+        ChangeSet { graph, indexer }
+    }
+
+    /// Batch insert unconfirmed transactions, filtering out those that are irrelevant.
+    ///
+    /// Relevancy is determined by the internal [`Indexer::is_tx_relevant`] implementation of `I`.
+    /// Irrelevant transactions in `txs` will be ignored.
+    ///
+    /// Items of `txs` are tuples containing the transaction and a *last seen* timestamp. The
+    /// *last seen* communicates when the transaction is last seen in the mempool which is used for
+    /// conflict-resolution in [`TxGraph`] (refer to [`TxGraph::insert_seen_at`] for details).
+    pub fn batch_insert_relevant_unconfirmed<'t>(
+        &mut self,
+        unconfirmed_txs: impl IntoIterator<Item = (&'t Transaction, u64)>,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        // The algorithm below allows for non-topologically ordered transactions by using two loops.
+        // This is achieved by:
+        // 1. insert all txs into the index. If they are irrelevant then that's fine it will just
+        //    not store anything about them.
+        // 2. decide whether to insert them into the graph depending on whether `is_tx_relevant`
+        //    returns true or not. (in a second loop).
+        let txs = unconfirmed_txs.into_iter().collect::<Vec<_>>();
+
+        let mut indexer = I::ChangeSet::default();
+        for (tx, _) in &txs {
+            indexer.append(self.index.index_tx(tx));
+        }
+
+        let graph = self.graph.batch_insert_unconfirmed(
+            txs.into_iter()
+                .filter(|(tx, _)| self.index.is_tx_relevant(tx))
+                .map(|(tx, seen_at)| (tx.clone(), seen_at)),
         );
-        additions
+
+        ChangeSet { graph, indexer }
+    }
+
+    /// Batch insert unconfirmed transactions.
+    ///
+    /// Items of `txs` are tuples containing the transaction and a *last seen* timestamp. The
+    /// *last seen* communicates when the transaction is last seen in the mempool which is used for
+    /// conflict-resolution in [`TxGraph`] (refer to [`TxGraph::insert_seen_at`] for details).
+    ///
+    /// To filter out irrelevant transactions, use [`batch_insert_relevant_unconfirmed`] instead.
+    ///
+    /// [`batch_insert_relevant_unconfirmed`]: IndexedTxGraph::batch_insert_relevant_unconfirmed
+    pub fn batch_insert_unconfirmed(
+        &mut self,
+        txs: impl IntoIterator<Item = (Transaction, u64)>,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        let graph = self.graph.batch_insert_unconfirmed(txs);
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
     }
 }
 
-/// A structure that represents changes to an [`IndexedTxGraph`].
+/// Methods are available if the anchor (`A`) implements [`AnchorFromBlockPosition`].
+impl<A: Anchor, I: Indexer> IndexedTxGraph<A, I>
+where
+    I::ChangeSet: Default + Append,
+    A: AnchorFromBlockPosition,
+{
+    /// Batch insert all transactions of the given `block` of `height`, filtering out those that are
+    /// irrelevant.
+    ///
+    /// Each inserted transaction's anchor will be constructed from
+    /// [`AnchorFromBlockPosition::from_block_position`].
+    ///
+    /// Relevancy is determined by the internal [`Indexer::is_tx_relevant`] implementation of `I`.
+    /// Irrelevant transactions in `txs` will be ignored.
+    pub fn apply_block_relevant(
+        &mut self,
+        block: &Block,
+        height: u32,
+    ) -> ChangeSet<A, I::ChangeSet> {
+        let block_id = BlockId {
+            hash: block.block_hash(),
+            height,
+        };
+        let mut changeset = ChangeSet::<A, I::ChangeSet>::default();
+        for (tx_pos, tx) in block.txdata.iter().enumerate() {
+            changeset.indexer.append(self.index.index_tx(tx));
+            if self.index.is_tx_relevant(tx) {
+                let txid = tx.compute_txid();
+                let anchor = A::from_block_position(block, block_id, tx_pos);
+                changeset.graph.append(self.graph.insert_tx(tx.clone()));
+                changeset
+                    .graph
+                    .append(self.graph.insert_anchor(txid, anchor));
+            }
+        }
+        changeset
+    }
+
+    /// Batch insert all transactions of the given `block` of `height`.
+    ///
+    /// Each inserted transaction's anchor will be constructed from
+    /// [`AnchorFromBlockPosition::from_block_position`].
+    ///
+    /// To only insert relevant transactions, use [`apply_block_relevant`] instead.
+    ///
+    /// [`apply_block_relevant`]: IndexedTxGraph::apply_block_relevant
+    pub fn apply_block(&mut self, block: Block, height: u32) -> ChangeSet<A, I::ChangeSet> {
+        let block_id = BlockId {
+            hash: block.block_hash(),
+            height,
+        };
+        let mut graph = tx_graph::ChangeSet::default();
+        for (tx_pos, tx) in block.txdata.iter().enumerate() {
+            let anchor = A::from_block_position(&block, block_id, tx_pos);
+            graph.append(self.graph.insert_anchor(tx.compute_txid(), anchor));
+            graph.append(self.graph.insert_tx(tx.clone()));
+        }
+        let indexer = self.index_tx_graph_changeset(&graph);
+        ChangeSet { graph, indexer }
+    }
+}
+
+/// Represents changes to an [`IndexedTxGraph`].
 #[derive(Clone, Debug, PartialEq)]
 #[cfg_attr(
     feature = "serde",
@@ -181,65 +283,78 @@ where
     )
 )]
 #[must_use]
-pub struct IndexedAdditions<A, IA> {
-    /// [`TxGraph`] additions.
-    pub graph_additions: Additions<A>,
-    /// [`Indexer`] additions.
-    pub index_additions: IA,
+pub struct ChangeSet<A, IA> {
+    /// [`TxGraph`] changeset.
+    pub graph: tx_graph::ChangeSet<A>,
+    /// [`Indexer`] changeset.
+    pub indexer: IA,
 }
 
-impl<A, IA: Default> Default for IndexedAdditions<A, IA> {
+impl<A, IA: Default> Default for ChangeSet<A, IA> {
     fn default() -> Self {
         Self {
-            graph_additions: Default::default(),
-            index_additions: Default::default(),
+            graph: Default::default(),
+            indexer: Default::default(),
         }
     }
 }
 
-impl<A: Anchor, IA: Append> Append for IndexedAdditions<A, IA> {
+impl<A: Anchor, IA: Append> Append for ChangeSet<A, IA> {
     fn append(&mut self, other: Self) {
-        self.graph_additions.append(other.graph_additions);
-        self.index_additions.append(other.index_additions);
+        self.graph.append(other.graph);
+        self.indexer.append(other.indexer);
     }
 
     fn is_empty(&self) -> bool {
-        self.graph_additions.is_empty() && self.index_additions.is_empty()
+        self.graph.is_empty() && self.indexer.is_empty()
     }
 }
 
-impl<A, IA: Default> From<Additions<A>> for IndexedAdditions<A, IA> {
-    fn from(graph_additions: Additions<A>) -> Self {
+impl<A, IA: Default> From<tx_graph::ChangeSet<A>> for ChangeSet<A, IA> {
+    fn from(graph: tx_graph::ChangeSet<A>) -> Self {
         Self {
-            graph_additions,
+            graph,
             ..Default::default()
         }
     }
 }
 
-impl<A, K> From<DerivationAdditions<K>> for IndexedAdditions<A, DerivationAdditions<K>> {
-    fn from(index_additions: DerivationAdditions<K>) -> Self {
+#[cfg(feature = "miniscript")]
+impl<A, K> From<crate::keychain::ChangeSet<K>> for ChangeSet<A, crate::keychain::ChangeSet<K>> {
+    fn from(indexer: crate::keychain::ChangeSet<K>) -> Self {
         Self {
-            graph_additions: Default::default(),
-            index_additions,
+            graph: Default::default(),
+            indexer,
         }
     }
 }
 
-/// Represents a structure that can index transaction data.
+/// Utilities for indexing transaction data.
+///
+/// Types which implement this trait can be used to construct an [`IndexedTxGraph`].
+/// This trait's methods should rarely be called directly.
 pub trait Indexer {
-    /// The resultant "additions" when new transaction data is indexed.
-    type Additions;
+    /// The resultant "changeset" when new transaction data is indexed.
+    type ChangeSet;
 
     /// Scan and index the given `outpoint` and `txout`.
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::Additions;
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet;
 
-    /// Scan and index the given transaction.
-    fn index_tx(&mut self, tx: &Transaction) -> Self::Additions;
+    /// Scans a transaction for relevant outpoints, which are stored and indexed internally.
+    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet;
 
-    /// Apply additions to itself.
-    fn apply_additions(&mut self, additions: Self::Additions);
+    /// Apply changeset to itself.
+    fn apply_changeset(&mut self, changeset: Self::ChangeSet);
+
+    /// Determines the [`ChangeSet`] between `self` and an empty [`Indexer`].
+    fn initial_changeset(&self) -> Self::ChangeSet;
 
     /// Determines whether the transaction should be included in the index.
     fn is_tx_relevant(&self, tx: &Transaction) -> bool;
 }
+
+impl<A, I> AsRef<TxGraph<A>> for IndexedTxGraph<A, I> {
+    fn as_ref(&self) -> &TxGraph<A> {
+        &self.graph
+    }
+}

crates/chain/src/keychain.rs

@@ -10,168 +10,12 @@
 //!
 //! [`SpkTxOutIndex`]: crate::SpkTxOutIndex
 
-use crate::{
-    collections::BTreeMap,
-    indexed_tx_graph::IndexedAdditions,
-    local_chain::{self, LocalChain},
-    tx_graph::TxGraph,
-    Anchor, Append,
-};
-
 #[cfg(feature = "miniscript")]
 mod txout_index;
+use bitcoin::{Amount, ScriptBuf};
 #[cfg(feature = "miniscript")]
 pub use txout_index::*;
 
-/// Represents updates to the derivation index of a [`KeychainTxOutIndex`].
-///
-/// It can be applied to [`KeychainTxOutIndex`] with [`apply_additions`]. [`DerivationAdditions] are
-/// monotone in that they will never decrease the revealed derivation index.
-///
-/// [`KeychainTxOutIndex`]: crate::keychain::KeychainTxOutIndex
-/// [`apply_additions`]: crate::keychain::KeychainTxOutIndex::apply_additions
-#[derive(Clone, Debug, PartialEq)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(
-        crate = "serde_crate",
-        bound(
-            deserialize = "K: Ord + serde::Deserialize<'de>",
-            serialize = "K: Ord + serde::Serialize"
-        )
-    )
-)]
-#[must_use]
-pub struct DerivationAdditions<K>(pub BTreeMap<K, u32>);
-
-impl<K> DerivationAdditions<K> {
-    /// Returns whether the additions are empty.
-    pub fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-
-    /// Get the inner map of the keychain to its new derivation index.
-    pub fn as_inner(&self) -> &BTreeMap<K, u32> {
-        &self.0
-    }
-}
-
-impl<K: Ord> Append for DerivationAdditions<K> {
-    /// Append another [`DerivationAdditions`] into self.
-    ///
-    /// If the keychain already exists, increase the index when the other's index > self's index.
-    /// If the keychain did not exist, append the new keychain.
-    fn append(&mut self, mut other: Self) {
-        self.0.iter_mut().for_each(|(key, index)| {
-            if let Some(other_index) = other.0.remove(key) {
-                *index = other_index.max(*index);
-            }
-        });
-
-        self.0.append(&mut other.0);
-    }
-
-    fn is_empty(&self) -> bool {
-        self.0.is_empty()
-    }
-}
-
-impl<K> Default for DerivationAdditions<K> {
-    fn default() -> Self {
-        Self(Default::default())
-    }
-}
-
-impl<K> AsRef<BTreeMap<K, u32>> for DerivationAdditions<K> {
-    fn as_ref(&self) -> &BTreeMap<K, u32> {
-        &self.0
-    }
-}
-
-/// A structure to update [`KeychainTxOutIndex`], [`TxGraph`] and [`LocalChain`]
-/// atomically.
-#[derive(Debug, Clone, PartialEq)]
-pub struct LocalUpdate<K, A> {
-    /// Last active derivation index per keychain (`K`).
-    pub keychain: BTreeMap<K, u32>,
-    /// Update for the [`TxGraph`].
-    pub graph: TxGraph<A>,
-    /// Update for the [`LocalChain`].
-    pub chain: LocalChain,
-}
-
-impl<K, A> Default for LocalUpdate<K, A> {
-    fn default() -> Self {
-        Self {
-            keychain: Default::default(),
-            graph: Default::default(),
-            chain: Default::default(),
-        }
-    }
-}
-
-/// A structure that records the corresponding changes as result of applying an [`LocalUpdate`].
-#[derive(Debug, Clone, PartialEq)]
-#[cfg_attr(
-    feature = "serde",
-    derive(serde::Deserialize, serde::Serialize),
-    serde(
-        crate = "serde_crate",
-        bound(
-            deserialize = "K: Ord + serde::Deserialize<'de>, A: Ord + serde::Deserialize<'de>",
-            serialize = "K: Ord + serde::Serialize, A: Ord + serde::Serialize",
-        )
-    )
-)]
-pub struct LocalChangeSet<K, A> {
-    /// Changes to the [`LocalChain`].
-    pub chain_changeset: local_chain::ChangeSet,
-
-    /// Additions to [`IndexedTxGraph`].
-    ///
-    /// [`IndexedTxGraph`]: crate::indexed_tx_graph::IndexedTxGraph
-    pub indexed_additions: IndexedAdditions<A, DerivationAdditions<K>>,
-}
-
-impl<K, A> Default for LocalChangeSet<K, A> {
-    fn default() -> Self {
-        Self {
-            chain_changeset: Default::default(),
-            indexed_additions: Default::default(),
-        }
-    }
-}
-
-impl<K: Ord, A: Anchor> Append for LocalChangeSet<K, A> {
-    fn append(&mut self, other: Self) {
-        Append::append(&mut self.chain_changeset, other.chain_changeset);
-        Append::append(&mut self.indexed_additions, other.indexed_additions);
-    }
-
-    fn is_empty(&self) -> bool {
-        self.chain_changeset.is_empty() && self.indexed_additions.is_empty()
-    }
-}
-
-impl<K, A> From<local_chain::ChangeSet> for LocalChangeSet<K, A> {
-    fn from(chain_changeset: local_chain::ChangeSet) -> Self {
-        Self {
-            chain_changeset,
-            ..Default::default()
-        }
-    }
-}
-
-impl<K, A> From<IndexedAdditions<A, DerivationAdditions<K>>> for LocalChangeSet<K, A> {
-    fn from(indexed_additions: IndexedAdditions<A, DerivationAdditions<K>>) -> Self {
-        Self {
-            indexed_additions,
-            ..Default::default()
-        }
-    }
-}
-
 /// Balance, differentiated into various categories.
 #[derive(Debug, PartialEq, Eq, Clone, Default)]
 #[cfg_attr(
@@ -181,13 +25,13 @@ impl<K, A> From<IndexedAdditions<A, DerivationAdditions<K>>> for LocalChangeSet<
 )]
 pub struct Balance {
     /// All coinbase outputs not yet matured
-    pub immature: u64,
+    pub immature: Amount,
     /// Unconfirmed UTXOs generated by a wallet tx
-    pub trusted_pending: u64,
+    pub trusted_pending: Amount,
     /// Unconfirmed UTXOs received from an external wallet
-    pub untrusted_pending: u64,
+    pub untrusted_pending: Amount,
     /// Confirmed and immediately spendable balance
-    pub confirmed: u64,
+    pub confirmed: Amount,
 }
 
 impl Balance {
@@ -195,16 +39,21 @@ impl Balance {
     ///
     /// This is the balance you can spend right now that shouldn't get cancelled via another party
     /// double spending it.
-    pub fn trusted_spendable(&self) -> u64 {
+    pub fn trusted_spendable(&self) -> Amount {
         self.confirmed + self.trusted_pending
     }
 
     /// Get the whole balance visible to the wallet.
-    pub fn total(&self) -> u64 {
+    pub fn total(&self) -> Amount {
         self.confirmed + self.trusted_pending + self.untrusted_pending + self.immature
     }
 }
 
+/// A tuple of keychain index and `T` representing the indexed value.
+pub type Indexed<T> = (u32, T);
+/// A tuple of keychain `K`, derivation index (`u32`) and a `T` associated with them.
+pub type KeychainIndexed<K, T> = ((K, u32), T);
+
 impl core::fmt::Display for Balance {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         write!(
@@ -227,40 +76,3 @@ impl core::ops::Add for Balance {
         }
     }
 }
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn append_keychain_derivation_indices() {
-        #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Debug)]
-        enum Keychain {
-            One,
-            Two,
-            Three,
-            Four,
-        }
-        let mut lhs_di = BTreeMap::<Keychain, u32>::default();
-        let mut rhs_di = BTreeMap::<Keychain, u32>::default();
-        lhs_di.insert(Keychain::One, 7);
-        lhs_di.insert(Keychain::Two, 0);
-        rhs_di.insert(Keychain::One, 3);
-        rhs_di.insert(Keychain::Two, 5);
-        lhs_di.insert(Keychain::Three, 3);
-        rhs_di.insert(Keychain::Four, 4);
-
-        let mut lhs = DerivationAdditions(lhs_di);
-        let rhs = DerivationAdditions(rhs_di);
-        lhs.append(rhs);
-
-        // Exiting index doesn't update if the new index in `other` is lower than `self`.
-        assert_eq!(lhs.0.get(&Keychain::One), Some(&7));
-        // Existing index updates if the new index in `other` is higher than `self`.
-        assert_eq!(lhs.0.get(&Keychain::Two), Some(&5));
-        // Existing index is unchanged if keychain doesn't exist in `other`.
-        assert_eq!(lhs.0.get(&Keychain::Three), Some(&3));
-        // New keychain gets added if the keychain is in `other` but not in `self`.
-        assert_eq!(lhs.0.get(&Keychain::Four), Some(&4));
-    }
-}

crates/chain/src/lib.rs

@@ -1,4 +1,4 @@
-//! This crate is a collection of core structures for [Bitcoin Dev Kit] (alpha release).
+//! This crate is a collection of core structures for [Bitcoin Dev Kit].
 //!
 //! The goal of this crate is to give wallets the mechanisms needed to:
 //!
@@ -12,9 +12,8 @@
 //!    you do it synchronously or asynchronously. If you know a fact about the blockchain, you can just
 //!    tell `bdk_chain`'s APIs about it, and that information will be integrated, if it can be done
 //!    consistently.
-//! 2. Error-free APIs.
-//! 3. Data persistence agnostic -- `bdk_chain` does not care where you cache on-chain data, what you
-//!    cache or how you fetch it.
+//! 2. Data persistence agnostic -- `bdk_chain` does not care where you cache on-chain data, what you
+//!    cache or how you retrieve it from persistent storage.
 //!
 //! [Bitcoin Dev Kit]: https://bitcoindevkit.org/
 
@@ -29,6 +28,7 @@ pub use chain_data::*;
 pub mod indexed_tx_graph;
 pub use indexed_tx_graph::IndexedTxGraph;
 pub mod keychain;
+pub use keychain::{Indexed, KeychainIndexed};
 pub mod local_chain;
 mod tx_data_traits;
 pub mod tx_graph;
@@ -36,8 +36,6 @@ pub use tx_data_traits::*;
 pub use tx_graph::TxGraph;
 mod chain_oracle;
 pub use chain_oracle::*;
-mod persist;
-pub use persist::*;
 
 #[doc(hidden)]
 pub mod example_utils;
@@ -47,11 +45,14 @@ pub use miniscript;
 #[cfg(feature = "miniscript")]
 mod descriptor_ext;
 #[cfg(feature = "miniscript")]
-pub use descriptor_ext::DescriptorExt;
+pub use descriptor_ext::{DescriptorExt, DescriptorId};
 #[cfg(feature = "miniscript")]
 mod spk_iter;
 #[cfg(feature = "miniscript")]
 pub use spk_iter::*;
+mod changeset;
+pub use changeset::*;
+pub mod spk_client;
 
 #[allow(unused_imports)]
 #[macro_use]
@@ -60,9 +61,6 @@ extern crate alloc;
 #[cfg(feature = "serde")]
 pub extern crate serde_crate as serde;
 
-#[cfg(feature = "bincode")]
-extern crate bincode;
-
 #[cfg(feature = "std")]
 #[macro_use]
 extern crate std;

crates/chain/src/persist.rs

@@ -1,97 +0,0 @@
-use core::convert::Infallible;
-
-use crate::Append;
-
-/// `Persist` wraps a [`PersistBackend`] (`B`) to create a convenient staging area for changes (`C`)
-/// before they are persisted.
-///
-/// Not all changes to the in-memory representation needs to be written to disk right away, so
-/// [`Persist::stage`] can be used to *stage* changes first and then [`Persist::commit`] can be used
-/// to write changes to disk.
-#[derive(Debug)]
-pub struct Persist<B, C> {
-    backend: B,
-    stage: C,
-}
-
-impl<B, C> Persist<B, C>
-where
-    B: PersistBackend<C>,
-    C: Default + Append,
-{
-    /// Create a new [`Persist`] from [`PersistBackend`].
-    pub fn new(backend: B) -> Self {
-        Self {
-            backend,
-            stage: Default::default(),
-        }
-    }
-
-    /// Stage a `changeset` to be commited later with [`commit`].
-    ///
-    /// [`commit`]: Self::commit
-    pub fn stage(&mut self, changeset: C) {
-        self.stage.append(changeset)
-    }
-
-    /// Get the changes that have not been commited yet.
-    pub fn staged(&self) -> &C {
-        &self.stage
-    }
-
-    /// Commit the staged changes to the underlying persistance backend.
-    ///
-    /// Changes that are committed (if any) are returned.
-    ///
-    /// # Error
-    ///
-    /// Returns a backend-defined error if this fails.
-    pub fn commit(&mut self) -> Result<Option<C>, B::WriteError> {
-        if self.stage.is_empty() {
-            return Ok(None);
-        }
-        self.backend
-            .write_changes(&self.stage)
-            // if written successfully, take and return `self.stage`
-            .map(|_| Some(core::mem::take(&mut self.stage)))
-    }
-}
-
-/// A persistence backend for [`Persist`].
-///
-/// `C` represents the changeset; a datatype that records changes made to in-memory data structures
-/// that are to be persisted, or retrieved from persistence.
-pub trait PersistBackend<C> {
-    /// The error the backend returns when it fails to write.
-    type WriteError: core::fmt::Debug;
-
-    /// The error the backend returns when it fails to load changesets `C`.
-    type LoadError: core::fmt::Debug;
-
-    /// Writes a changeset to the persistence backend.
-    ///
-    /// It is up to the backend what it does with this. It could store every changeset in a list or
-    /// it inserts the actual changes into a more structured database. All it needs to guarantee is
-    /// that [`load_from_persistence`] restores a keychain tracker to what it should be if all
-    /// changesets had been applied sequentially.
-    ///
-    /// [`load_from_persistence`]: Self::load_from_persistence
-    fn write_changes(&mut self, changeset: &C) -> Result<(), Self::WriteError>;
-
-    /// Return the aggregate changeset `C` from persistence.
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError>;
-}
-
-impl<C: Default> PersistBackend<C> for () {
-    type WriteError = Infallible;
-
-    type LoadError = Infallible;
-
-    fn write_changes(&mut self, _changeset: &C) -> Result<(), Self::WriteError> {
-        Ok(())
-    }
-
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError> {
-        Ok(C::default())
-    }
-}

crates/chain/src/spk_client.rs

@@ -0,0 +1,389 @@
+//! Helper types for spk-based blockchain clients.
+
+use crate::{
+    collections::BTreeMap, keychain::Indexed, local_chain::CheckPoint,
+    ConfirmationTimeHeightAnchor, TxGraph,
+};
+use alloc::boxed::Box;
+use bitcoin::{OutPoint, Script, ScriptBuf, Txid};
+use core::marker::PhantomData;
+
+/// Data required to perform a spk-based blockchain client sync.
+///
+/// A client sync fetches relevant chain data for a known list of scripts, transaction ids and
+/// outpoints. The sync process also updates the chain from the given [`CheckPoint`].
+pub struct SyncRequest {
+    /// A checkpoint for the current chain [`LocalChain::tip`].
+    /// The sync process will return a new chain update that extends this tip.
+    ///
+    /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
+    pub chain_tip: CheckPoint,
+    /// Transactions that spend from or to these indexed script pubkeys.
+    pub spks: Box<dyn ExactSizeIterator<Item = ScriptBuf> + Send>,
+    /// Transactions with these txids.
+    pub txids: Box<dyn ExactSizeIterator<Item = Txid> + Send>,
+    /// Transactions with these outpoints or spent from these outpoints.
+    pub outpoints: Box<dyn ExactSizeIterator<Item = OutPoint> + Send>,
+}
+
+impl SyncRequest {
+    /// Construct a new [`SyncRequest`] from a given `cp` tip.
+    pub fn from_chain_tip(cp: CheckPoint) -> Self {
+        Self {
+            chain_tip: cp,
+            spks: Box::new(core::iter::empty()),
+            txids: Box::new(core::iter::empty()),
+            outpoints: Box::new(core::iter::empty()),
+        }
+    }
+
+    /// Set the [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_spks(
+        mut self,
+        spks: impl IntoIterator<IntoIter = impl ExactSizeIterator<Item = ScriptBuf> + Send + 'static>,
+    ) -> Self {
+        self.spks = Box::new(spks.into_iter());
+        self
+    }
+
+    /// Set the [`Txid`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_txids(
+        mut self,
+        txids: impl IntoIterator<IntoIter = impl ExactSizeIterator<Item = Txid> + Send + 'static>,
+    ) -> Self {
+        self.txids = Box::new(txids.into_iter());
+        self
+    }
+
+    /// Set the [`OutPoint`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_outpoints(
+        mut self,
+        outpoints: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = OutPoint> + Send + 'static,
+        >,
+    ) -> Self {
+        self.outpoints = Box::new(outpoints.into_iter());
+        self
+    }
+
+    /// Chain on additional [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_spks(
+        mut self,
+        spks: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = ScriptBuf> + Send + 'static,
+            Item = ScriptBuf,
+        >,
+    ) -> Self {
+        self.spks = Box::new(ExactSizeChain::new(self.spks, spks.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`Txid`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_txids(
+        mut self,
+        txids: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = Txid> + Send + 'static,
+            Item = Txid,
+        >,
+    ) -> Self {
+        self.txids = Box::new(ExactSizeChain::new(self.txids, txids.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`OutPoint`]s that will be synced against.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_outpoints(
+        mut self,
+        outpoints: impl IntoIterator<
+            IntoIter = impl ExactSizeIterator<Item = OutPoint> + Send + 'static,
+            Item = OutPoint,
+        >,
+    ) -> Self {
+        self.outpoints = Box::new(ExactSizeChain::new(self.outpoints, outpoints.into_iter()));
+        self
+    }
+
+    /// Add a closure that will be called for [`Script`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks(
+        mut self,
+        mut inspect: impl FnMut(&Script) + Send + Sync + 'static,
+    ) -> Self {
+        self.spks = Box::new(self.spks.inspect(move |spk| inspect(spk)));
+        self
+    }
+
+    /// Add a closure that will be called for [`Txid`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_txids(mut self, mut inspect: impl FnMut(&Txid) + Send + Sync + 'static) -> Self {
+        self.txids = Box::new(self.txids.inspect(move |txid| inspect(txid)));
+        self
+    }
+
+    /// Add a closure that will be called for [`OutPoint`]s previously added to this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_outpoints(
+        mut self,
+        mut inspect: impl FnMut(&OutPoint) + Send + Sync + 'static,
+    ) -> Self {
+        self.outpoints = Box::new(self.outpoints.inspect(move |op| inspect(op)));
+        self
+    }
+
+    /// Populate the request with revealed script pubkeys from `index` with the given `spk_range`.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[cfg(feature = "miniscript")]
+    #[must_use]
+    pub fn populate_with_revealed_spks<K: Clone + Ord + core::fmt::Debug + Send + Sync>(
+        self,
+        index: &crate::keychain::KeychainTxOutIndex<K>,
+        spk_range: impl core::ops::RangeBounds<K>,
+    ) -> Self {
+        use alloc::borrow::ToOwned;
+        use alloc::vec::Vec;
+        self.chain_spks(
+            index
+                .revealed_spks(spk_range)
+                .map(|(_, spk)| spk.to_owned())
+                .collect::<Vec<_>>(),
+        )
+    }
+}
+
+/// Data returned from a spk-based blockchain client sync.
+///
+/// See also [`SyncRequest`].
+pub struct SyncResult<A = ConfirmationTimeHeightAnchor> {
+    /// The update to apply to the receiving [`TxGraph`].
+    pub graph_update: TxGraph<A>,
+    /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain).
+    pub chain_update: CheckPoint,
+}
+
+/// Data required to perform a spk-based blockchain client full scan.
+///
+/// A client full scan iterates through all the scripts for the given keychains, fetching relevant
+/// data until some stop gap number of scripts is found that have no data. This operation is
+/// generally only used when importing or restoring previously used keychains in which the list of
+/// used scripts is not known. The full scan process also updates the chain from the given [`CheckPoint`].
+pub struct FullScanRequest<K> {
+    /// A checkpoint for the current [`LocalChain::tip`].
+    /// The full scan process will return a new chain update that extends this tip.
+    ///
+    /// [`LocalChain::tip`]: crate::local_chain::LocalChain::tip
+    pub chain_tip: CheckPoint,
+    /// Iterators of script pubkeys indexed by the keychain index.
+    pub spks_by_keychain: BTreeMap<K, Box<dyn Iterator<Item = Indexed<ScriptBuf>> + Send>>,
+}
+
+impl<K: Ord + Clone> FullScanRequest<K> {
+    /// Construct a new [`FullScanRequest`] from a given `chain_tip`.
+    #[must_use]
+    pub fn from_chain_tip(chain_tip: CheckPoint) -> Self {
+        Self {
+            chain_tip,
+            spks_by_keychain: BTreeMap::new(),
+        }
+    }
+
+    /// Construct a new [`FullScanRequest`] from a given `chain_tip` and `index`.
+    ///
+    /// Unbounded script pubkey iterators for each keychain (`K`) are extracted using
+    /// [`KeychainTxOutIndex::all_unbounded_spk_iters`] and is used to populate the
+    /// [`FullScanRequest`].
+    ///
+    /// [`KeychainTxOutIndex::all_unbounded_spk_iters`]: crate::keychain::KeychainTxOutIndex::all_unbounded_spk_iters
+    #[cfg(feature = "miniscript")]
+    #[must_use]
+    pub fn from_keychain_txout_index(
+        chain_tip: CheckPoint,
+        index: &crate::keychain::KeychainTxOutIndex<K>,
+    ) -> Self
+    where
+        K: core::fmt::Debug,
+    {
+        let mut req = Self::from_chain_tip(chain_tip);
+        for (keychain, spks) in index.all_unbounded_spk_iters() {
+            req = req.set_spks_for_keychain(keychain, spks);
+        }
+        req
+    }
+
+    /// Set the [`Script`]s for a given `keychain`.
+    ///
+    /// This consumes the [`FullScanRequest`] and returns the updated one.
+    #[must_use]
+    pub fn set_spks_for_keychain(
+        mut self,
+        keychain: K,
+        spks: impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send + 'static>,
+    ) -> Self {
+        self.spks_by_keychain
+            .insert(keychain, Box::new(spks.into_iter()));
+        self
+    }
+
+    /// Chain on additional [`Script`]s that will be synced against.
+    ///
+    /// This consumes the [`FullScanRequest`] and returns the updated one.
+    #[must_use]
+    pub fn chain_spks_for_keychain(
+        mut self,
+        keychain: K,
+        spks: impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send + 'static>,
+    ) -> Self {
+        match self.spks_by_keychain.remove(&keychain) {
+            // clippy here suggests to remove `into_iter` from `spks.into_iter()`, but doing so
+            // results in a compilation error
+            #[allow(clippy::useless_conversion)]
+            Some(keychain_spks) => self
+                .spks_by_keychain
+                .insert(keychain, Box::new(keychain_spks.chain(spks.into_iter()))),
+            None => self
+                .spks_by_keychain
+                .insert(keychain, Box::new(spks.into_iter())),
+        };
+        self
+    }
+
+    /// Add a closure that will be called for every [`Script`] previously added to any keychain in
+    /// this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks_for_all_keychains(
+        mut self,
+        inspect: impl FnMut(K, u32, &Script) + Send + Sync + Clone + 'static,
+    ) -> Self
+    where
+        K: Send + 'static,
+    {
+        for (keychain, spks) in core::mem::take(&mut self.spks_by_keychain) {
+            let mut inspect = inspect.clone();
+            self.spks_by_keychain.insert(
+                keychain.clone(),
+                Box::new(spks.inspect(move |(i, spk)| inspect(keychain.clone(), *i, spk))),
+            );
+        }
+        self
+    }
+
+    /// Add a closure that will be called for every [`Script`] previously added to a given
+    /// `keychain` in this request.
+    ///
+    /// This consumes the [`SyncRequest`] and returns the updated one.
+    #[must_use]
+    pub fn inspect_spks_for_keychain(
+        mut self,
+        keychain: K,
+        mut inspect: impl FnMut(u32, &Script) + Send + Sync + 'static,
+    ) -> Self
+    where
+        K: Send + 'static,
+    {
+        if let Some(spks) = self.spks_by_keychain.remove(&keychain) {
+            self.spks_by_keychain.insert(
+                keychain,
+                Box::new(spks.inspect(move |(i, spk)| inspect(*i, spk))),
+            );
+        }
+        self
+    }
+}
+
+/// Data returned from a spk-based blockchain client full scan.
+///
+/// See also [`FullScanRequest`].
+pub struct FullScanResult<K, A = ConfirmationTimeHeightAnchor> {
+    /// The update to apply to the receiving [`LocalChain`](crate::local_chain::LocalChain).
+    pub graph_update: TxGraph<A>,
+    /// The update to apply to the receiving [`TxGraph`].
+    pub chain_update: CheckPoint,
+    /// Last active indices for the corresponding keychains (`K`).
+    pub last_active_indices: BTreeMap<K, u32>,
+}
+
+/// A version of [`core::iter::Chain`] which can combine two [`ExactSizeIterator`]s to form a new
+/// [`ExactSizeIterator`].
+///
+/// The danger of this is explained in [the `ExactSizeIterator` docs]
+/// (https://doc.rust-lang.org/core/iter/trait.ExactSizeIterator.html#when-shouldnt-an-adapter-be-exactsizeiterator).
+/// This does not apply here since it would be impossible to scan an item count that overflows
+/// `usize` anyway.
+struct ExactSizeChain<A, B, I> {
+    a: Option<A>,
+    b: Option<B>,
+    i: PhantomData<I>,
+}
+
+impl<A, B, I> ExactSizeChain<A, B, I> {
+    fn new(a: A, b: B) -> Self {
+        ExactSizeChain {
+            a: Some(a),
+            b: Some(b),
+            i: PhantomData,
+        }
+    }
+}
+
+impl<A, B, I> Iterator for ExactSizeChain<A, B, I>
+where
+    A: Iterator<Item = I>,
+    B: Iterator<Item = I>,
+{
+    type Item = I;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if let Some(a) = &mut self.a {
+            let item = a.next();
+            if item.is_some() {
+                return item;
+            }
+            self.a = None;
+        }
+        if let Some(b) = &mut self.b {
+            let item = b.next();
+            if item.is_some() {
+                return item;
+            }
+            self.b = None;
+        }
+        None
+    }
+}
+
+impl<A, B, I> ExactSizeIterator for ExactSizeChain<A, B, I>
+where
+    A: ExactSizeIterator<Item = I>,
+    B: ExactSizeIterator<Item = I>,
+{
+    fn len(&self) -> usize {
+        let a_len = self.a.as_ref().map(|a| a.len()).unwrap_or(0);
+        let b_len = self.b.as_ref().map(|a| a.len()).unwrap_or(0);
+        a_len + b_len
+    }
+}

crates/chain/src/spk_iter.rs

@@ -1,5 +1,6 @@
 use crate::{
-    bitcoin::{secp256k1::Secp256k1, Script},
+    bitcoin::{secp256k1::Secp256k1, ScriptBuf},
+    keychain::Indexed,
     miniscript::{Descriptor, DescriptorPublicKey},
 };
 use core::{borrow::Borrow, ops::Bound, ops::RangeBounds};
@@ -22,9 +23,9 @@ pub const BIP32_MAX_INDEX: u32 = (1 << 31) - 1;
 /// # use std::str::FromStr;
 /// # let secp = bitcoin::secp256k1::Secp256k1::signing_only();
 /// # let (descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
-/// # let external_spk_0 = descriptor.at_derivation_index(0).script_pubkey();
-/// # let external_spk_3 = descriptor.at_derivation_index(3).script_pubkey();
-/// # let external_spk_4 = descriptor.at_derivation_index(4).script_pubkey();
+/// # let external_spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
+/// # let external_spk_3 = descriptor.at_derivation_index(3).unwrap().script_pubkey();
+/// # let external_spk_4 = descriptor.at_derivation_index(4).unwrap().script_pubkey();
 ///
 /// // Creates a new script pubkey iterator starting at 0 from a descriptor.
 /// let mut spk_iter = SpkIterator::new(&descriptor);
@@ -43,48 +44,61 @@ impl<D> SpkIterator<D>
 where
     D: Borrow<Descriptor<DescriptorPublicKey>>,
 {
-    /// Creates a new script pubkey iterator starting at 0 from a descriptor.
+    /// Create a new script pubkey iterator from `descriptor`.
+    ///
+    /// This iterates from derivation index 0 and stops at index 0x7FFFFFFF (as specified in
+    /// BIP-32). Non-wildcard descriptors will only return one script pubkey at derivation index 0.
+    ///
+    /// Use [`new_with_range`](SpkIterator::new_with_range) to create an iterator with a specified
+    /// derivation index range.
     pub fn new(descriptor: D) -> Self {
-        let end = if descriptor.borrow().has_wildcard() {
-            BIP32_MAX_INDEX
-        } else {
-            0
-        };
-
-        SpkIterator::new_with_range(descriptor, 0..=end)
+        SpkIterator::new_with_range(descriptor, 0..=BIP32_MAX_INDEX)
     }
 
-    // Creates a new script pubkey iterator from a descriptor with a given range.
-    pub(crate) fn new_with_range<R>(descriptor: D, range: R) -> Self
+    /// Create a new script pubkey iterator from `descriptor` and a given `range`.
+    ///
+    /// Non-wildcard descriptors will only emit a single script pubkey (at derivation index 0).
+    /// Wildcard descriptors have an end-bound of 0x7FFFFFFF (inclusive).
+    ///
+    /// Refer to [`new`](SpkIterator::new) for more.
+    pub fn new_with_range<R>(descriptor: D, range: R) -> Self
     where
         R: RangeBounds<u32>,
     {
+        let start = match range.start_bound() {
+            Bound::Included(start) => *start,
+            Bound::Excluded(start) => *start + 1,
+            Bound::Unbounded => u32::MIN,
+        };
+
         let mut end = match range.end_bound() {
             Bound::Included(end) => *end + 1,
             Bound::Excluded(end) => *end,
             Bound::Unbounded => u32::MAX,
         };
+
         // Because `end` is exclusive, we want the maximum value to be BIP32_MAX_INDEX + 1.
         end = end.min(BIP32_MAX_INDEX + 1);
 
         Self {
-            next_index: match range.start_bound() {
-                Bound::Included(start) => *start,
-                Bound::Excluded(start) => *start + 1,
-                Bound::Unbounded => u32::MIN,
-            },
+            next_index: start,
             end,
             descriptor,
             secp: Secp256k1::verification_only(),
         }
     }
+
+    /// Get a reference to the internal descriptor.
+    pub fn descriptor(&self) -> &D {
+        &self.descriptor
+    }
 }
 
 impl<D> Iterator for SpkIterator<D>
 where
     D: Borrow<Descriptor<DescriptorPublicKey>>,
 {
-    type Item = (u32, Script);
+    type Item = Indexed<ScriptBuf>;
 
     fn next(&mut self) -> Option<Self::Item> {
         // For non-wildcard descriptors, we expect the first element to be Some((0, spk)), then None after.
@@ -93,11 +107,15 @@ where
             return None;
         }
 
+        // If the descriptor is non-wildcard, only index 0 will return an spk.
+        if !self.descriptor.borrow().has_wildcard() && self.next_index != 0 {
+            return None;
+        }
+
         let script = self
             .descriptor
             .borrow()
-            .at_derivation_index(self.next_index)
-            .derived_descriptor(&self.secp)
+            .derived_descriptor(&self.secp, self.next_index)
             .expect("the descriptor cannot need hardened derivation")
             .script_pubkey();
         let output = (self.next_index, script);
@@ -135,45 +153,48 @@ mod test {
         Descriptor<DescriptorPublicKey>,
         Descriptor<DescriptorPublicKey>,
     ) {
-        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+        let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
         let secp = Secp256k1::signing_only();
         let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
         let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
 
-        txout_index.add_keychain(TestKeychain::External, external_descriptor.clone());
-        txout_index.add_keychain(TestKeychain::Internal, internal_descriptor.clone());
+        let _ = txout_index
+            .insert_descriptor(TestKeychain::External, external_descriptor.clone())
+            .unwrap();
+        let _ = txout_index
+            .insert_descriptor(TestKeychain::Internal, internal_descriptor.clone())
+            .unwrap();
 
         (txout_index, external_descriptor, internal_descriptor)
     }
 
     #[test]
     #[allow(clippy::iter_nth_zero)]
+    #[rustfmt::skip]
     fn test_spkiterator_wildcard() {
         let (_, external_desc, _) = init_txout_index();
-        let external_spk_0 = external_desc.at_derivation_index(0).script_pubkey();
-        let external_spk_16 = external_desc.at_derivation_index(16).script_pubkey();
-        let external_spk_20 = external_desc.at_derivation_index(20).script_pubkey();
-        let external_spk_21 = external_desc.at_derivation_index(21).script_pubkey();
-        let external_spk_max = external_desc
-            .at_derivation_index(BIP32_MAX_INDEX)
-            .script_pubkey();
+        let external_spk_0 = external_desc.at_derivation_index(0).unwrap().script_pubkey();
+        let external_spk_16 = external_desc.at_derivation_index(16).unwrap().script_pubkey();
+        let external_spk_20 = external_desc.at_derivation_index(20).unwrap().script_pubkey();
+        let external_spk_21 = external_desc.at_derivation_index(21).unwrap().script_pubkey();
+        let external_spk_max = external_desc.at_derivation_index(BIP32_MAX_INDEX).unwrap().script_pubkey();
 
         let mut external_spk = SpkIterator::new(&external_desc);
         let max_index = BIP32_MAX_INDEX - 22;
 
-        assert_eq!(external_spk.next().unwrap(), (0, external_spk_0));
-        assert_eq!(external_spk.nth(15).unwrap(), (16, external_spk_16));
-        assert_eq!(external_spk.nth(3).unwrap(), (20, external_spk_20.clone()));
-        assert_eq!(external_spk.next().unwrap(), (21, external_spk_21));
+        assert_eq!(external_spk.next(), Some((0, external_spk_0)));
+        assert_eq!(external_spk.nth(15), Some((16, external_spk_16)));
+        assert_eq!(external_spk.nth(3), Some((20, external_spk_20.clone())));
+        assert_eq!(external_spk.next(), Some((21, external_spk_21)));
         assert_eq!(
-            external_spk.nth(max_index as usize).unwrap(),
-            (BIP32_MAX_INDEX, external_spk_max)
+            external_spk.nth(max_index as usize),
+            Some((BIP32_MAX_INDEX, external_spk_max))
         );
         assert_eq!(external_spk.nth(0), None);
 
         let mut external_spk = SpkIterator::new_with_range(&external_desc, 0..21);
-        assert_eq!(external_spk.nth(20).unwrap(), (20, external_spk_20));
+        assert_eq!(external_spk.nth(20), Some((20, external_spk_20)));
         assert_eq!(external_spk.next(), None);
 
         let mut external_spk = SpkIterator::new_with_range(&external_desc, 0..21);
@@ -187,29 +208,65 @@ mod test {
         let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
         let external_spk_0 = no_wildcard_descriptor
             .at_derivation_index(0)
+            .unwrap()
             .script_pubkey();
 
         let mut external_spk = SpkIterator::new(&no_wildcard_descriptor);
 
-        assert_eq!(external_spk.next().unwrap(), (0, external_spk_0.clone()));
+        assert_eq!(external_spk.next(), Some((0, external_spk_0.clone())));
         assert_eq!(external_spk.next(), None);
 
         let mut external_spk = SpkIterator::new(&no_wildcard_descriptor);
 
-        assert_eq!(external_spk.nth(0).unwrap(), (0, external_spk_0));
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0.clone())));
         assert_eq!(external_spk.nth(0), None);
-    }
 
-    // The following dummy traits were created to test if SpkIterator is working properly.
-    trait TestSendStatic: Send + 'static {
-        fn test(&self) -> u32 {
-            20
-        }
-    }
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..0);
 
-    impl TestSendStatic for SpkIterator<Descriptor<DescriptorPublicKey>> {
-        fn test(&self) -> u32 {
-            20
-        }
+        assert_eq!(external_spk.next(), None);
+
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..1);
+
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0.clone())));
+        assert_eq!(external_spk.next(), None);
+
+        // We test that using new_with_range with range_len > 1 gives back an iterator with
+        // range_len = 1
+        let mut external_spk = SpkIterator::new_with_range(&no_wildcard_descriptor, 0..10);
+
+        assert_eq!(external_spk.nth(0), Some((0, external_spk_0)));
+        assert_eq!(external_spk.nth(0), None);
+
+        // non index-0 should NOT return an spk
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..1).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..=1).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..2).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 1..=2).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 10..11).next(),
+            None
+        );
+        assert_eq!(
+            SpkIterator::new_with_range(&no_wildcard_descriptor, 10..=10).next(),
+            None
+        );
     }
 }
+
+#[test]
+fn spk_iterator_is_send_and_static() {
+    fn is_send_and_static<A: Send + 'static>() {}
+    is_send_and_static::<SpkIterator<Descriptor<DescriptorPublicKey>>>()
+}

crates/chain/src/spk_txout_index.rs

@@ -3,15 +3,15 @@ use core::ops::RangeBounds;
 use crate::{
     collections::{hash_map::Entry, BTreeMap, BTreeSet, HashMap},
     indexed_tx_graph::Indexer,
-    ForEachTxOut,
 };
-use bitcoin::{self, OutPoint, Script, Transaction, TxOut, Txid};
+use bitcoin::{Amount, OutPoint, Script, ScriptBuf, SignedAmount, Transaction, TxOut, Txid};
 
 /// An index storing [`TxOut`]s that have a script pubkey that matches those in a list.
 ///
 /// The basic idea is that you insert script pubkeys you care about into the index with
-/// [`insert_spk`] and then when you call [`scan`], the index will look at any txouts you pass in and
-/// store and index any txouts matching one of its script pubkeys.
+/// [`insert_spk`] and then when you call [`Indexer::index_tx`] or [`Indexer::index_txout`], the
+/// index will look at any txouts you pass in and store and index any txouts matching one of its
+/// script pubkeys.
 ///
 /// Each script pubkey is associated with an application-defined index script index `I`, which must be
 /// [`Ord`]. Usually, this is used to associate the derivation index of the script pubkey or even a
@@ -25,14 +25,13 @@ use bitcoin::{self, OutPoint, Script, Transaction, TxOut, Txid};
 /// [`TxOut`]: bitcoin::TxOut
 /// [`insert_spk`]: Self::insert_spk
 /// [`Ord`]: core::cmp::Ord
-/// [`scan`]: Self::scan
 /// [`TxGraph`]: crate::tx_graph::TxGraph
 #[derive(Clone, Debug)]
 pub struct SpkTxOutIndex<I> {
     /// script pubkeys ordered by index
-    spks: BTreeMap<I, Script>,
+    spks: BTreeMap<I, ScriptBuf>,
     /// A reverse lookup from spk to spk index
-    spk_indices: HashMap<Script, I>,
+    spk_indices: HashMap<ScriptBuf, I>,
     /// The set of unused indexes.
     unused: BTreeSet<I>,
     /// Lookup index and txout by outpoint.
@@ -53,20 +52,22 @@ impl<I> Default for SpkTxOutIndex<I> {
     }
 }
 
-impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
-    type Additions = ();
+impl<I: Clone + Ord + core::fmt::Debug> Indexer for SpkTxOutIndex<I> {
+    type ChangeSet = ();
 
-    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::Additions {
+    fn index_txout(&mut self, outpoint: OutPoint, txout: &TxOut) -> Self::ChangeSet {
         self.scan_txout(outpoint, txout);
         Default::default()
     }
 
-    fn index_tx(&mut self, tx: &Transaction) -> Self::Additions {
+    fn index_tx(&mut self, tx: &Transaction) -> Self::ChangeSet {
         self.scan(tx);
         Default::default()
     }
 
-    fn apply_additions(&mut self, _additions: Self::Additions) {
+    fn initial_changeset(&self) -> Self::ChangeSet {}
+
+    fn apply_changeset(&mut self, _changeset: Self::ChangeSet) {
         // This applies nothing.
     }
 
@@ -75,41 +76,23 @@ impl<I: Clone + Ord> Indexer for SpkTxOutIndex<I> {
     }
 }
 
-/// This macro is used instead of a member function of `SpkTxOutIndex`, which would result in a
-/// compiler error[E0521]: "borrowed data escapes out of closure" when we attempt to take a
-/// reference out of the `ForEachTxOut` closure during scanning.
-macro_rules! scan_txout {
-    ($self:ident, $op:expr, $txout:expr) => {{
-        let spk_i = $self.spk_indices.get(&$txout.script_pubkey);
-        if let Some(spk_i) = spk_i {
-            $self.txouts.insert($op, (spk_i.clone(), $txout.clone()));
-            $self.spk_txouts.insert((spk_i.clone(), $op));
-            $self.unused.remove(&spk_i);
-        }
-        spk_i
-    }};
-}
-
-impl<I: Clone + Ord> SpkTxOutIndex<I> {
-    /// Scans an object containing many txouts.
+impl<I: Clone + Ord + core::fmt::Debug> SpkTxOutIndex<I> {
+    /// Scans a transaction's outputs for matching script pubkeys.
     ///
     /// Typically, this is used in two situations:
     ///
     /// 1. After loading transaction data from the disk, you may scan over all the txouts to restore all
     /// your txouts.
     /// 2. When getting new data from the chain, you usually scan it before incorporating it into your chain state.
-    ///
-    /// See [`ForEachTxout`] for the types that support this.
-    ///
-    /// [`ForEachTxout`]: crate::ForEachTxOut
-    pub fn scan(&mut self, txouts: &impl ForEachTxOut) -> BTreeSet<I> {
+    pub fn scan(&mut self, tx: &Transaction) -> BTreeSet<I> {
         let mut scanned_indices = BTreeSet::new();
-
-        txouts.for_each_txout(|(op, txout)| {
-            if let Some(spk_i) = scan_txout!(self, op, txout) {
+        let txid = tx.compute_txid();
+        for (i, txout) in tx.output.iter().enumerate() {
+            let op = OutPoint::new(txid, i as u32);
+            if let Some(spk_i) = self.scan_txout(op, txout) {
                 scanned_indices.insert(spk_i.clone());
             }
-        });
+        }
 
         scanned_indices
     }
@@ -117,7 +100,13 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Scan a single `TxOut` for a matching script pubkey and returns the index that matches the
     /// script pubkey (if any).
     pub fn scan_txout(&mut self, op: OutPoint, txout: &TxOut) -> Option<&I> {
-        scan_txout!(self, op, txout)
+        let spk_i = self.spk_indices.get(&txout.script_pubkey);
+        if let Some(spk_i) = spk_i {
+            self.txouts.insert(op, (spk_i.clone(), txout.clone()));
+            self.spk_txouts.insert((spk_i.clone(), op));
+            self.unused.remove(spk_i);
+        }
+        spk_i
     }
 
     /// Get a reference to the set of indexed outpoints.
@@ -152,11 +141,11 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
         use bitcoin::hashes::Hash;
         use core::ops::Bound::*;
         let min_op = OutPoint {
-            txid: Txid::from_inner([0x00; 32]),
+            txid: Txid::all_zeros(),
             vout: u32::MIN,
         };
         let max_op = OutPoint {
-            txid: Txid::from_inner([0xff; 32]),
+            txid: Txid::from_byte_array([0xff; Txid::LEN]),
             vout: u32::MAX,
         };
 
@@ -179,27 +168,25 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     ///
     /// Returns `None` if the `TxOut` hasn't been scanned or if nothing matching was found there.
     pub fn txout(&self, outpoint: OutPoint) -> Option<(&I, &TxOut)> {
-        self.txouts
-            .get(&outpoint)
-            .map(|(spk_i, txout)| (spk_i, txout))
+        self.txouts.get(&outpoint).map(|v| (&v.0, &v.1))
     }
 
     /// Returns the script that has been inserted at the `index`.
     ///
     /// If that index hasn't been inserted yet, it will return `None`.
     pub fn spk_at_index(&self, index: &I) -> Option<&Script> {
-        self.spks.get(index)
+        self.spks.get(index).map(|s| s.as_script())
     }
 
     /// The script pubkeys that are being tracked by the index.
-    pub fn all_spks(&self) -> &BTreeMap<I, Script> {
+    pub fn all_spks(&self) -> &BTreeMap<I, ScriptBuf> {
         &self.spks
     }
 
     /// Adds a script pubkey to scan for. Returns `false` and does nothing if spk already exists in the map
     ///
     /// the index will look for outputs spending to this spk whenever it scans new data.
-    pub fn insert_spk(&mut self, index: I, spk: Script) -> bool {
+    pub fn insert_spk(&mut self, index: I, spk: ScriptBuf) -> bool {
         match self.spk_indices.entry(spk.clone()) {
             Entry::Vacant(value) => {
                 value.insert(index.clone());
@@ -228,7 +215,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// let unused_change_spks =
     ///     txout_index.unused_spks((change_index, u32::MIN)..(change_index, u32::MAX));
     /// ```
-    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)>
+    pub fn unused_spks<R>(&self, range: R) -> impl DoubleEndedIterator<Item = (&I, &Script)> + Clone
     where
         R: RangeBounds<I>,
     {
@@ -242,7 +229,7 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
     /// Here, "unused" means that after the script pubkey was stored in the index, the index has
     /// never scanned a transaction output with it.
     pub fn is_used(&self, index: &I) -> bool {
-        self.unused.get(index).is_none()
+        !self.unused.contains(index)
     }
 
     /// Marks the script pubkey at `index` as used even though it hasn't seen an output spending to it.
@@ -283,37 +270,45 @@ impl<I: Clone + Ord> SpkTxOutIndex<I> {
         self.spk_indices.get(script)
     }
 
-    /// Computes total input value going from script pubkeys in the index (sent) and the total output
-    /// value going to script pubkeys in the index (received) in `tx`. For the `sent` to be computed
-    /// correctly, the output being spent must have already been scanned by the index. Calculating
-    /// received just uses the transaction outputs directly, so it will be correct even if it has not
-    /// been scanned.
-    pub fn sent_and_received(&self, tx: &Transaction) -> (u64, u64) {
-        let mut sent = 0;
-        let mut received = 0;
+    /// Computes the total value transfer effect `tx` has on the script pubkeys in `range`. Value is
+    /// *sent* when a script pubkey in the `range` is on an input and *received* when it is on an
+    /// output. For `sent` to be computed correctly, the output being spent must have already been
+    /// scanned by the index. Calculating received just uses the [`Transaction`] outputs directly,
+    /// so it will be correct even if it has not been scanned.
+    pub fn sent_and_received(
+        &self,
+        tx: &Transaction,
+        range: impl RangeBounds<I>,
+    ) -> (Amount, Amount) {
+        let mut sent = Amount::ZERO;
+        let mut received = Amount::ZERO;
 
         for txin in &tx.input {
-            if let Some((_, txout)) = self.txout(txin.previous_output) {
-                sent += txout.value;
+            if let Some((index, txout)) = self.txout(txin.previous_output) {
+                if range.contains(index) {
+                    sent += txout.value;
+                }
             }
         }
         for txout in &tx.output {
-            if self.index_of_spk(&txout.script_pubkey).is_some() {
-                received += txout.value;
+            if let Some(index) = self.index_of_spk(&txout.script_pubkey) {
+                if range.contains(index) {
+                    received += txout.value;
+                }
             }
         }
 
         (sent, received)
     }
 
-    /// Computes the net value that this transaction gives to the script pubkeys in the index and
-    /// *takes* from the transaction outputs in the index. Shorthand for calling
-    /// [`sent_and_received`] and subtracting sent from received.
+    /// Computes the net value transfer effect of `tx` on the script pubkeys in `range`. Shorthand
+    /// for calling [`sent_and_received`] and subtracting sent from received.
     ///
     /// [`sent_and_received`]: Self::sent_and_received
-    pub fn net_value(&self, tx: &Transaction) -> i64 {
-        let (sent, received) = self.sent_and_received(tx);
-        received as i64 - sent as i64
+    pub fn net_value(&self, tx: &Transaction, range: impl RangeBounds<I>) -> SignedAmount {
+        let (sent, received) = self.sent_and_received(tx, range);
+        received.to_signed().expect("valid `SignedAmount`")
+            - sent.to_signed().expect("valid `SignedAmount`")
     }
 
     /// Whether any of the inputs of this transaction spend a txout tracked or whether any output

crates/chain/src/tx_data_traits.rs

@@ -2,46 +2,91 @@ use crate::collections::BTreeMap;
 use crate::collections::BTreeSet;
 use crate::BlockId;
 use alloc::vec::Vec;
-use bitcoin::{Block, OutPoint, Transaction, TxOut};
-
-/// Trait to do something with every txout contained in a structure.
-///
-/// We would prefer to just work with things that can give us an `Iterator<Item=(OutPoint, &TxOut)>`
-/// here, but rust's type system makes it extremely hard to do this (without trait objects).
-pub trait ForEachTxOut {
-    /// The provided closure `f` will be called with each `outpoint/txout` pair.
-    fn for_each_txout(&self, f: impl FnMut((OutPoint, &TxOut)));
-}
-
-impl ForEachTxOut for Block {
-    fn for_each_txout(&self, mut f: impl FnMut((OutPoint, &TxOut))) {
-        for tx in self.txdata.iter() {
-            tx.for_each_txout(&mut f)
-        }
-    }
-}
-
-impl ForEachTxOut for Transaction {
-    fn for_each_txout(&self, mut f: impl FnMut((OutPoint, &TxOut))) {
-        let txid = self.txid();
-        for (i, txout) in self.output.iter().enumerate() {
-            f((
-                OutPoint {
-                    txid,
-                    vout: i as u32,
-                },
-                txout,
-            ))
-        }
-    }
-}
 
 /// Trait that "anchors" blockchain data to a specific block of height and hash.
 ///
-/// I.e. If transaction A is anchored in block B, then if block B is in the best chain, we can
+/// If transaction A is anchored in block B, and block B is in the best chain, we can
 /// assume that transaction A is also confirmed in the best chain. This does not necessarily mean
 /// that transaction A is confirmed in block B. It could also mean transaction A is confirmed in a
 /// parent block of B.
+///
+/// Every [`Anchor`] implementation must contain a [`BlockId`] parameter, and must implement
+/// [`Ord`]. When implementing [`Ord`], the anchors' [`BlockId`]s should take precedence
+/// over other elements inside the [`Anchor`]s for comparison purposes, i.e., you should first
+/// compare the anchors' [`BlockId`]s and then care about the rest.
+///
+/// The example shows different types of anchors:
+/// ```
+/// # use bdk_chain::local_chain::LocalChain;
+/// # use bdk_chain::tx_graph::TxGraph;
+/// # use bdk_chain::BlockId;
+/// # use bdk_chain::ConfirmationHeightAnchor;
+/// # use bdk_chain::ConfirmationTimeHeightAnchor;
+/// # use bdk_chain::example_utils::*;
+/// # use bitcoin::hashes::Hash;
+/// // Initialize the local chain with two blocks.
+/// let chain = LocalChain::from_blocks(
+///     [
+///         (1, Hash::hash("first".as_bytes())),
+///         (2, Hash::hash("second".as_bytes())),
+///     ]
+///     .into_iter()
+///     .collect(),
+/// );
+///
+/// // Transaction to be inserted into `TxGraph`s with different anchor types.
+/// let tx = tx_from_hex(RAW_TX_1);
+///
+/// // Insert `tx` into a `TxGraph` that uses `BlockId` as the anchor type.
+/// // When a transaction is anchored with `BlockId`, the anchor block and the confirmation block of
+/// // the transaction is the same block.
+/// let mut graph_a = TxGraph::<BlockId>::default();
+/// let _ = graph_a.insert_tx(tx.clone());
+/// graph_a.insert_anchor(
+///     tx.compute_txid(),
+///     BlockId {
+///         height: 1,
+///         hash: Hash::hash("first".as_bytes()),
+///     },
+/// );
+///
+/// // Insert `tx` into a `TxGraph` that uses `ConfirmationHeightAnchor` as the anchor type.
+/// // This anchor records the anchor block and the confirmation height of the transaction.
+/// // When a transaction is anchored with `ConfirmationHeightAnchor`, the anchor block and
+/// // confirmation block can be different. However, the confirmation block cannot be higher than
+/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
+/// let mut graph_b = TxGraph::<ConfirmationHeightAnchor>::default();
+/// let _ = graph_b.insert_tx(tx.clone());
+/// graph_b.insert_anchor(
+///     tx.compute_txid(),
+///     ConfirmationHeightAnchor {
+///         anchor_block: BlockId {
+///             height: 2,
+///             hash: Hash::hash("second".as_bytes()),
+///         },
+///         confirmation_height: 1,
+///     },
+/// );
+///
+/// // Insert `tx` into a `TxGraph` that uses `ConfirmationTimeHeightAnchor` as the anchor type.
+/// // This anchor records the anchor block, the confirmation height and time of the transaction.
+/// // When a transaction is anchored with `ConfirmationTimeHeightAnchor`, the anchor block and
+/// // confirmation block can be different. However, the confirmation block cannot be higher than
+/// // the anchor block and both blocks must be in the same chain for the anchor to be valid.
+/// let mut graph_c = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+/// let _ = graph_c.insert_tx(tx.clone());
+/// graph_c.insert_anchor(
+///     tx.compute_txid(),
+///     ConfirmationTimeHeightAnchor {
+///         anchor_block: BlockId {
+///             height: 2,
+///             hash: Hash::hash("third".as_bytes()),
+///         },
+///         confirmation_height: 1,
+///         confirmation_time: 123,
+///     },
+/// );
+/// ```
 pub trait Anchor: core::fmt::Debug + Clone + Eq + PartialOrd + Ord + core::hash::Hash {
     /// Returns the [`BlockId`] that the associated blockchain data is "anchored" in.
     fn anchor_block(&self) -> BlockId;
@@ -55,32 +100,42 @@ pub trait Anchor: core::fmt::Debug + Clone + Eq + PartialOrd + Ord + core::hash:
     }
 }
 
-impl<A: Anchor> Anchor for &'static A {
+impl<'a, A: Anchor> Anchor for &'a A {
     fn anchor_block(&self) -> BlockId {
         <A as Anchor>::anchor_block(self)
     }
 }
 
+/// An [`Anchor`] that can be constructed from a given block, block height and transaction position
+/// within the block.
+pub trait AnchorFromBlockPosition: Anchor {
+    /// Construct the anchor from a given `block`, block height and `tx_pos` within the block.
+    fn from_block_position(block: &bitcoin::Block, block_id: BlockId, tx_pos: usize) -> Self;
+}
+
 /// Trait that makes an object appendable.
-pub trait Append {
+pub trait Append: Default {
     /// Append another object of the same type onto `self`.
     fn append(&mut self, other: Self);
 
     /// Returns whether the structure is considered empty.
     fn is_empty(&self) -> bool;
-}
 
-impl Append for () {
-    fn append(&mut self, _other: Self) {}
-
-    fn is_empty(&self) -> bool {
-        true
+    /// Take the value, replacing it with the default value.
+    fn take(&mut self) -> Option<Self> {
+        if self.is_empty() {
+            None
+        } else {
+            Some(core::mem::take(self))
+        }
     }
 }
 
 impl<K: Ord, V> Append for BTreeMap<K, V> {
-    fn append(&mut self, mut other: Self) {
-        BTreeMap::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeMap::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {
@@ -89,8 +144,10 @@ impl<K: Ord, V> Append for BTreeMap<K, V> {
 }
 
 impl<T: Ord> Append for BTreeSet<T> {
-    fn append(&mut self, mut other: Self) {
-        BTreeSet::append(self, &mut other)
+    fn append(&mut self, other: Self) {
+        // We use `extend` instead of `BTreeMap::append` due to performance issues with `append`.
+        // Refer to https://github.com/rust-lang/rust/issues/34666#issuecomment-675658420
+        BTreeSet::extend(self, other)
     }
 
     fn is_empty(&self) -> bool {
@@ -108,13 +165,30 @@ impl<T> Append for Vec<T> {
     }
 }
 
-impl<A: Append, B: Append> Append for (A, B) {
-    fn append(&mut self, other: Self) {
-        Append::append(&mut self.0, other.0);
-        Append::append(&mut self.1, other.1);
-    }
+macro_rules! impl_append_for_tuple {
+    ($($a:ident $b:tt)*) => {
+        impl<$($a),*> Append for ($($a,)*) where $($a: Append),* {
 
-    fn is_empty(&self) -> bool {
-        Append::is_empty(&self.0) && Append::is_empty(&self.1)
+            fn append(&mut self, _other: Self) {
+                $(Append::append(&mut self.$b, _other.$b) );*
+            }
+
+            fn is_empty(&self) -> bool {
+                $(Append::is_empty(&self.$b) && )* true
+            }
+        }
     }
 }
+
+impl_append_for_tuple!();
+impl_append_for_tuple!(T0 0);
+impl_append_for_tuple!(T0 0 T1 1);
+impl_append_for_tuple!(T0 0 T1 1 T2 2);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9);
+impl_append_for_tuple!(T0 0 T1 1 T2 2 T3 3 T4 4 T5 5 T6 6 T7 7 T8 8 T9 9 T10 10);

crates/chain/tests/common/mod.rs

@@ -1,3 +1,19 @@
+#![cfg(feature = "miniscript")]
+
+mod tx_template;
+#[allow(unused_imports)]
+pub use tx_template::*;
+
+#[allow(unused_macros)]
+macro_rules! block_id {
+    ($height:expr, $hash:literal) => {{
+        bdk_chain::BlockId {
+            height: $height,
+            hash: bitcoin::hashes::Hash::hash($hash.as_bytes()),
+        }
+    }};
+}
+
 #[allow(unused_macros)]
 macro_rules! h {
     ($index:literal) => {{
@@ -9,25 +25,18 @@ macro_rules! h {
 macro_rules! local_chain {
     [ $(($height:expr, $block_hash:expr)), * ] => {{
         #[allow(unused_mut)]
-        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*])
+        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect())
+            .expect("chain must have genesis block")
     }};
 }
 
 #[allow(unused_macros)]
-macro_rules! chain {
-    ($([$($tt:tt)*]),*) => { chain!( checkpoints: [$([$($tt)*]),*] ) };
-    (checkpoints: $($tail:tt)*) => { chain!( index: TxHeight, checkpoints: $($tail)*) };
-    (index: $ind:ty, checkpoints: [ $([$height:expr, $block_hash:expr]),* ] $(,txids: [$(($txid:expr, $tx_height:expr)),*])?) => {{
+macro_rules! chain_update {
+    [ $(($height:expr, $hash:expr)), * ] => {{
         #[allow(unused_mut)]
-        let mut chain = bdk_chain::sparse_chain::SparseChain::<$ind>::from_checkpoints([$(($height, $block_hash).into()),*]);
-
-        $(
-            $(
-                let _ = chain.insert_tx($txid, $tx_height).expect("should succeed");
-            )*
-        )?
-
-        chain
+        bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $hash).into()),*].into_iter().collect())
+            .expect("chain must have genesis block")
+            .tip()
     }};
 }
 
@@ -60,9 +69,21 @@ macro_rules! changeset {
 #[allow(unused)]
 pub fn new_tx(lt: u32) -> bitcoin::Transaction {
     bitcoin::Transaction {
-        version: 0x00,
-        lock_time: bitcoin::PackedLockTime(lt),
+        version: bitcoin::transaction::Version::non_standard(0x00),
+        lock_time: bitcoin::absolute::LockTime::from_consensus(lt),
         input: vec![],
         output: vec![],
     }
 }
+
+#[allow(unused)]
+pub const DESCRIPTORS: [&str; 7] = [
+    "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)",
+    "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)",
+    "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0/*)",
+    "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)",
+    "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)",
+    "wpkh(xprv9s21ZrQH143K4EXURwMHuLS469fFzZyXk7UUpdKfQwhoHcAiYTakpe8pMU2RiEdvrU9McyuE7YDoKcXkoAwEGoK53WBDnKKv2zZbb9BzttX/1/0/*)",
+    // non-wildcard
+    "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)",
+];

crates/chain/tests/common/tx_template.rs

@@ -0,0 +1,139 @@
+#![cfg(feature = "miniscript")]
+
+use rand::distributions::{Alphanumeric, DistString};
+use std::collections::HashMap;
+
+use bdk_chain::{tx_graph::TxGraph, Anchor, SpkTxOutIndex};
+use bitcoin::{
+    locktime::absolute::LockTime, secp256k1::Secp256k1, transaction, Amount, OutPoint, ScriptBuf,
+    Sequence, Transaction, TxIn, TxOut, Txid, Witness,
+};
+use miniscript::Descriptor;
+
+/// Template for creating a transaction in `TxGraph`.
+///
+/// The incentive for transaction templates is to create a transaction history in a simple manner to
+/// avoid having to explicitly hash previous transactions to form previous outpoints of later
+/// transactions.
+#[derive(Clone, Copy, Default)]
+pub struct TxTemplate<'a, A> {
+    /// Uniquely identifies the transaction, before it can have a txid.
+    pub tx_name: &'a str,
+    pub inputs: &'a [TxInTemplate<'a>],
+    pub outputs: &'a [TxOutTemplate],
+    pub anchors: &'a [A],
+    pub last_seen: Option<u64>,
+}
+
+#[allow(dead_code)]
+pub enum TxInTemplate<'a> {
+    /// This will give a random txid and vout.
+    Bogus,
+
+    /// This is used for coinbase transactions because they do not have previous outputs.
+    Coinbase,
+
+    /// Contains the `tx_name` and `vout` that we are spending. The rule is that we must only spend
+    /// from tx of a previous `TxTemplate`.
+    PrevTx(&'a str, usize),
+}
+
+pub struct TxOutTemplate {
+    pub value: u64,
+    pub spk_index: Option<u32>, // some = get spk from SpkTxOutIndex, none = random spk
+}
+
+#[allow(unused)]
+impl TxOutTemplate {
+    pub fn new(value: u64, spk_index: Option<u32>) -> Self {
+        TxOutTemplate { value, spk_index }
+    }
+}
+
+#[allow(dead_code)]
+pub fn init_graph<'a, A: Anchor + Clone + 'a>(
+    tx_templates: impl IntoIterator<Item = &'a TxTemplate<'a, A>>,
+) -> (TxGraph<A>, SpkTxOutIndex<u32>, HashMap<&'a str, Txid>) {
+    let (descriptor, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), super::DESCRIPTORS[2]).unwrap();
+    let mut graph = TxGraph::<A>::default();
+    let mut spk_index = SpkTxOutIndex::default();
+    (0..10).for_each(|index| {
+        spk_index.insert_spk(
+            index,
+            descriptor
+                .at_derivation_index(index)
+                .unwrap()
+                .script_pubkey(),
+        );
+    });
+    let mut tx_ids = HashMap::<&'a str, Txid>::new();
+
+    for (bogus_txin_vout, tx_tmp) in tx_templates.into_iter().enumerate() {
+        let tx = Transaction {
+            version: transaction::Version::non_standard(0),
+            lock_time: LockTime::ZERO,
+            input: tx_tmp
+                .inputs
+                .iter()
+                .map(|input| match input {
+                    TxInTemplate::Bogus => TxIn {
+                        previous_output: OutPoint::new(
+                            bitcoin::hashes::Hash::hash(
+                                Alphanumeric
+                                    .sample_string(&mut rand::thread_rng(), 20)
+                                    .as_bytes(),
+                            ),
+                            bogus_txin_vout as u32,
+                        ),
+                        script_sig: ScriptBuf::new(),
+                        sequence: Sequence::default(),
+                        witness: Witness::new(),
+                    },
+                    TxInTemplate::Coinbase => TxIn {
+                        previous_output: OutPoint::null(),
+                        script_sig: ScriptBuf::new(),
+                        sequence: Sequence::MAX,
+                        witness: Witness::new(),
+                    },
+                    TxInTemplate::PrevTx(prev_name, prev_vout) => {
+                        let prev_txid = tx_ids.get(prev_name).expect(
+                            "txin template must spend from tx of template that comes before",
+                        );
+                        TxIn {
+                            previous_output: OutPoint::new(*prev_txid, *prev_vout as _),
+                            script_sig: ScriptBuf::new(),
+                            sequence: Sequence::default(),
+                            witness: Witness::new(),
+                        }
+                    }
+                })
+                .collect(),
+            output: tx_tmp
+                .outputs
+                .iter()
+                .map(|output| match &output.spk_index {
+                    None => TxOut {
+                        value: Amount::from_sat(output.value),
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    Some(index) => TxOut {
+                        value: Amount::from_sat(output.value),
+                        script_pubkey: spk_index.spk_at_index(index).unwrap().to_owned(),
+                    },
+                })
+                .collect(),
+        };
+
+        tx_ids.insert(tx_tmp.tx_name, tx.compute_txid());
+        spk_index.scan(&tx);
+        let _ = graph.insert_tx(tx.clone());
+        for anchor in tx_tmp.anchors.iter() {
+            let _ = graph.insert_anchor(tx.compute_txid(), anchor.clone());
+        }
+        if let Some(seen_at) = tx_tmp.last_seen {
+            let _ = graph.insert_seen_at(tx.compute_txid(), seen_at);
+        }
+    }
+    (graph, spk_index, tx_ids)
+}

crates/chain/tests/test_indexed_tx_graph.rs

@@ -1,16 +1,20 @@
+#![cfg(feature = "miniscript")]
+
 #[macro_use]
 mod common;
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::{collections::BTreeSet, sync::Arc};
 
+use crate::common::DESCRIPTORS;
 use bdk_chain::{
-    indexed_tx_graph::{IndexedAdditions, IndexedTxGraph},
-    keychain::{Balance, DerivationAdditions, KeychainTxOutIndex},
+    indexed_tx_graph::{self, IndexedTxGraph},
+    keychain::{self, Balance, KeychainTxOutIndex},
     local_chain::LocalChain,
-    tx_graph::Additions,
-    BlockId, ChainPosition, ConfirmationHeightAnchor,
+    tx_graph, Append, ChainPosition, ConfirmationHeightAnchor, DescriptorExt,
+};
+use bitcoin::{
+    secp256k1::Secp256k1, Amount, OutPoint, Script, ScriptBuf, Transaction, TxIn, TxOut,
 };
-use bitcoin::{secp256k1::Secp256k1, BlockHash, OutPoint, Script, Transaction, TxIn, TxOut};
 use miniscript::Descriptor;
 
 /// Ensure [`IndexedTxGraph::insert_relevant_txs`] can successfully index transactions NOT presented
@@ -22,24 +26,27 @@ use miniscript::Descriptor;
 /// agnostic.
 #[test]
 fn insert_relevant_txs() {
-    const DESCRIPTOR: &str = "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)";
-    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTOR)
+    let (descriptor, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), DESCRIPTORS[0])
         .expect("must be valid");
-    let spk_0 = descriptor.at_derivation_index(0).script_pubkey();
-    let spk_1 = descriptor.at_derivation_index(9).script_pubkey();
+    let spk_0 = descriptor.at_derivation_index(0).unwrap().script_pubkey();
+    let spk_1 = descriptor.at_derivation_index(9).unwrap().script_pubkey();
 
-    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::default();
-    graph.index.add_keychain((), descriptor);
-    graph.index.set_lookahead(&(), 10);
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<()>>::new(
+        KeychainTxOutIndex::new(10),
+    );
+    let _ = graph
+        .index
+        .insert_descriptor((), descriptor.clone())
+        .unwrap();
 
     let tx_a = Transaction {
         output: vec![
             TxOut {
-                value: 10_000,
+                value: Amount::from_sat(10_000),
                 script_pubkey: spk_0,
             },
             TxOut {
-                value: 20_000,
+                value: Amount::from_sat(20_000),
                 script_pubkey: spk_1,
             },
         ],
@@ -48,7 +55,7 @@ fn insert_relevant_txs() {
 
     let tx_b = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx_a.txid(), 0),
+            previous_output: OutPoint::new(tx_a.compute_txid(), 0),
             ..Default::default()
         }],
         ..common::new_tx(1)
@@ -56,7 +63,7 @@ fn insert_relevant_txs() {
 
     let tx_c = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx_a.txid(), 1),
+            previous_output: OutPoint::new(tx_a.compute_txid(), 1),
             ..Default::default()
         }],
         ..common::new_tx(2)
@@ -64,19 +71,34 @@ fn insert_relevant_txs() {
 
     let txs = [tx_c, tx_b, tx_a];
 
+    let changeset = indexed_tx_graph::ChangeSet {
+        graph: tx_graph::ChangeSet {
+            txs: txs.iter().cloned().map(Arc::new).collect(),
+            ..Default::default()
+        },
+        indexer: keychain::ChangeSet {
+            last_revealed: [(descriptor.descriptor_id(), 9_u32)].into(),
+            keychains_added: [].into(),
+        },
+    };
+
     assert_eq!(
-        graph.insert_relevant_txs(txs.iter().map(|tx| (tx, None)), None),
-        IndexedAdditions {
-            graph_additions: Additions {
-                txs: txs.into(),
-                ..Default::default()
-            },
-            index_additions: DerivationAdditions([((), 9_u32)].into()),
-        }
-    )
+        graph.batch_insert_relevant(txs.iter().map(|tx| (tx, None))),
+        changeset,
+    );
+
+    // The initial changeset will also contain info about the keychain we added
+    let initial_changeset = indexed_tx_graph::ChangeSet {
+        graph: changeset.graph,
+        indexer: keychain::ChangeSet {
+            last_revealed: changeset.indexer.last_revealed,
+            keychains_added: [((), descriptor)].into(),
+        },
+    };
+
+    assert_eq!(graph.initial_changeset(), initial_changeset);
 }
 
-#[test]
 /// Ensure consistency IndexedTxGraph list_* and balance methods. These methods lists
 /// relevant txouts and utxos from the information fetched from a ChainOracle (here a LocalChain).
 ///
@@ -104,44 +126,57 @@ fn insert_relevant_txs() {
 ///
 /// Finally Add more blocks to local chain until tx1 coinbase maturity hits.
 /// Assert maturity at coinbase maturity inflection height. Block height 98 and 99.
-
+#[test]
 fn test_list_owned_txouts() {
     // Create Local chains
-
-    let local_chain = (0..150)
-        .map(|i| (i as u32, h!("random")))
-        .collect::<BTreeMap<u32, BlockHash>>();
-    let local_chain = LocalChain::from(local_chain);
+    let local_chain = LocalChain::from_blocks((0..150).map(|i| (i as u32, h!("random"))).collect())
+        .expect("must have genesis hash");
 
     // Initiate IndexedTxGraph
 
-    let (desc_1, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/0/*)").unwrap();
-    let (desc_2, _) = Descriptor::parse_descriptor(&Secp256k1::signing_only(), "tr(tprv8ZgxMBicQKsPd3krDUsBAmtnRsK3rb8u5yi1zhQgMhF1tR8MW7xfE4rnrbbsrbPR52e7rKapu6ztw1jXveJSCGHEriUGZV7mCe88duLp5pj/86'/1'/0'/1/*)").unwrap();
+    let (desc_1, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), common::DESCRIPTORS[2]).unwrap();
+    let (desc_2, _) =
+        Descriptor::parse_descriptor(&Secp256k1::signing_only(), common::DESCRIPTORS[3]).unwrap();
 
-    let mut graph =
-        IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::default();
+    let mut graph = IndexedTxGraph::<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>::new(
+        KeychainTxOutIndex::new(10),
+    );
 
-    graph.index.add_keychain("keychain_1".into(), desc_1);
-    graph.index.add_keychain("keychain_2".into(), desc_2);
-    graph.index.set_lookahead_for_all(10);
+    assert!(!graph
+        .index
+        .insert_descriptor("keychain_1".into(), desc_1)
+        .unwrap()
+        .is_empty());
+    assert!(!graph
+        .index
+        .insert_descriptor("keychain_2".into(), desc_2)
+        .unwrap()
+        .is_empty());
 
     // Get trusted and untrusted addresses
 
-    let mut trusted_spks = Vec::new();
-    let mut untrusted_spks = Vec::new();
+    let mut trusted_spks: Vec<ScriptBuf> = Vec::new();
+    let mut untrusted_spks: Vec<ScriptBuf> = Vec::new();
 
     {
         // we need to scope here to take immutanble reference of the graph
         for _ in 0..10 {
-            let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_1".to_string());
+            let ((_, script), _) = graph
+                .index
+                .reveal_next_spk(&"keychain_1".to_string())
+                .unwrap();
             // TODO Assert indexes
-            trusted_spks.push(script.clone());
+            trusted_spks.push(script.to_owned());
         }
     }
     {
         for _ in 0..10 {
-            let ((_, script), _) = graph.index.reveal_next_spk(&"keychain_2".to_string());
-            untrusted_spks.push(script.clone());
+            let ((_, script), _) = graph
+                .index
+                .reveal_next_spk(&"keychain_2".to_string())
+                .unwrap();
+            untrusted_spks.push(script.to_owned());
         }
     }
 
@@ -154,8 +189,8 @@ fn test_list_owned_txouts() {
             ..Default::default()
         }],
         output: vec![TxOut {
-            value: 70000,
-            script_pubkey: trusted_spks[0].clone(),
+            value: Amount::from_sat(70000),
+            script_pubkey: trusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -163,8 +198,8 @@ fn test_list_owned_txouts() {
     // tx2 is an incoming transaction received at untrusted keychain at block 1.
     let tx2 = Transaction {
         output: vec![TxOut {
-            value: 30000,
-            script_pubkey: untrusted_spks[0].clone(),
+            value: Amount::from_sat(30000),
+            script_pubkey: untrusted_spks[0].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -172,12 +207,12 @@ fn test_list_owned_txouts() {
     // tx3 spends tx2 and gives a change back in trusted keychain. Confirmed at Block 2.
     let tx3 = Transaction {
         input: vec![TxIn {
-            previous_output: OutPoint::new(tx2.txid(), 0),
+            previous_output: OutPoint::new(tx2.compute_txid(), 0),
             ..Default::default()
         }],
         output: vec![TxOut {
-            value: 10000,
-            script_pubkey: trusted_spks[1].clone(),
+            value: Amount::from_sat(10000),
+            script_pubkey: trusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -185,8 +220,8 @@ fn test_list_owned_txouts() {
     // tx4 is an external transaction receiving at untrusted keychain, unconfirmed.
     let tx4 = Transaction {
         output: vec![TxOut {
-            value: 20000,
-            script_pubkey: untrusted_spks[1].clone(),
+            value: Amount::from_sat(20000),
+            script_pubkey: untrusted_spks[1].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -194,8 +229,8 @@ fn test_list_owned_txouts() {
     // tx5 is spending tx3 and receiving change at trusted keychain, unconfirmed.
     let tx5 = Transaction {
         output: vec![TxOut {
-            value: 15000,
-            script_pubkey: trusted_spks[2].clone(),
+            value: Amount::from_sat(15000),
+            script_pubkey: trusted_spks[2].to_owned(),
         }],
         ..common::new_tx(0)
     };
@@ -206,35 +241,31 @@ fn test_list_owned_txouts() {
     // Insert transactions into graph with respective anchors
     // For unconfirmed txs we pass in `None`.
 
-    let _ = graph.insert_relevant_txs(
-        [&tx1, &tx2, &tx3, &tx6].iter().enumerate().map(|(i, tx)| {
+    let _ =
+        graph.batch_insert_relevant([&tx1, &tx2, &tx3, &tx6].iter().enumerate().map(|(i, tx)| {
             let height = i as u32;
             (
                 *tx,
                 local_chain
-                    .blocks()
-                    .get(&height)
-                    .map(|&hash| BlockId { height, hash })
+                    .get(height)
+                    .map(|cp| cp.block_id())
                     .map(|anchor_block| ConfirmationHeightAnchor {
                         anchor_block,
                         confirmation_height: anchor_block.height,
                     }),
             )
-        }),
-        None,
-    );
+        }));
 
-    let _ = graph.insert_relevant_txs([&tx4, &tx5].iter().map(|tx| (*tx, None)), Some(100));
+    let _ = graph.batch_insert_relevant_unconfirmed([&tx4, &tx5].iter().map(|tx| (*tx, 100)));
 
     // A helper lambda to extract and filter data from the graph.
     let fetch =
         |height: u32,
          graph: &IndexedTxGraph<ConfirmationHeightAnchor, KeychainTxOutIndex<String>>| {
             let chain_tip = local_chain
-                .blocks()
-                .get(&height)
-                .map(|&hash| BlockId { height, hash })
-                .expect("block must exist");
+                .get(height)
+                .map(|cp| cp.block_id())
+                .unwrap_or_else(|| panic!("block must exist at {}", height));
             let txouts = graph
                 .graph()
                 .filter_chain_txouts(
@@ -257,7 +288,7 @@ fn test_list_owned_txouts() {
                 &local_chain,
                 chain_tip,
                 graph.index.outpoints().iter().cloned(),
-                |_, spk: &Script| trusted_spks.contains(spk),
+                |_, spk: &Script| trusted_spks.contains(&spk.to_owned()),
             );
 
             assert_eq!(txouts.len(), 5);
@@ -328,25 +359,31 @@ fn test_list_owned_txouts() {
             balance,
         ) = fetch(0, &graph);
 
-        assert_eq!(confirmed_txouts_txid, [tx1.txid()].into());
+        assert_eq!(confirmed_txouts_txid, [tx1.compute_txid()].into());
         assert_eq!(
             unconfirmed_txouts_txid,
-            [tx2.txid(), tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [
+                tx2.compute_txid(),
+                tx3.compute_txid(),
+                tx4.compute_txid(),
+                tx5.compute_txid()
+            ]
+            .into()
         );
 
-        assert_eq!(confirmed_utxos_txid, [tx1.txid()].into());
+        assert_eq!(confirmed_utxos_txid, [tx1.compute_txid()].into());
         assert_eq!(
             unconfirmed_utxos_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx3.compute_txid(), tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 25000,   // tx3 + tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 0              // Nothing is confirmed yet
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(25000),   // tx3 + tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::ZERO                     // Nothing is confirmed yet
             }
         );
     }
@@ -362,26 +399,29 @@ fn test_list_owned_txouts() {
         ) = fetch(1, &graph);
 
         // tx2 gets into confirmed txout set
-        assert_eq!(confirmed_txouts_txid, [tx1.txid(), tx2.txid()].into());
+        assert_eq!(
+            confirmed_txouts_txid,
+            [tx1.compute_txid(), tx2.compute_txid()].into()
+        );
         assert_eq!(
             unconfirmed_txouts_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx3.compute_txid(), tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
         // tx2 doesn't get into confirmed utxos set
-        assert_eq!(confirmed_utxos_txid, [tx1.txid()].into());
+        assert_eq!(confirmed_utxos_txid, [tx1.compute_txid()].into());
         assert_eq!(
             unconfirmed_utxos_txid,
-            [tx3.txid(), tx4.txid(), tx5.txid()].into()
+            [tx3.compute_txid(), tx4.compute_txid(), tx5.compute_txid()].into()
         );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 25000,   // tx3 + tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 0              // Nothing is confirmed yet
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(25000),   // tx3 + tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::ZERO                     // Nothing is confirmed yet
             }
         );
     }
@@ -399,21 +439,30 @@ fn test_list_owned_txouts() {
         // tx3 now gets into the confirmed txout set
         assert_eq!(
             confirmed_txouts_txid,
-            [tx1.txid(), tx2.txid(), tx3.txid()].into()
+            [tx1.compute_txid(), tx2.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_txouts_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
-        assert_eq!(unconfirmed_txouts_txid, [tx4.txid(), tx5.txid()].into());
 
         // tx3 also gets into confirmed utxo set
-        assert_eq!(confirmed_utxos_txid, [tx1.txid(), tx3.txid()].into());
-        assert_eq!(unconfirmed_utxos_txid, [tx4.txid(), tx5.txid()].into());
+        assert_eq!(
+            confirmed_utxos_txid,
+            [tx1.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_utxos_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
+        );
 
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 10000          // tx3 got confirmed
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(10000)          // tx3 got confirmed
             }
         );
     }
@@ -430,21 +479,30 @@ fn test_list_owned_txouts() {
 
         assert_eq!(
             confirmed_txouts_txid,
-            [tx1.txid(), tx2.txid(), tx3.txid()].into()
+            [tx1.compute_txid(), tx2.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_txouts_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
         );
-        assert_eq!(unconfirmed_txouts_txid, [tx4.txid(), tx5.txid()].into());
 
-        assert_eq!(confirmed_utxos_txid, [tx1.txid(), tx3.txid()].into());
-        assert_eq!(unconfirmed_utxos_txid, [tx4.txid(), tx5.txid()].into());
+        assert_eq!(
+            confirmed_utxos_txid,
+            [tx1.compute_txid(), tx3.compute_txid()].into()
+        );
+        assert_eq!(
+            unconfirmed_utxos_txid,
+            [tx4.compute_txid(), tx5.compute_txid()].into()
+        );
 
         // Coinbase is still immature
         assert_eq!(
             balance,
             Balance {
-                immature: 70000,          // immature coinbase
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 10000          // tx1 got matured
+                immature: Amount::from_sat(70000),          // immature coinbase
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(10000)          // tx1 got matured
             }
         );
     }
@@ -457,10 +515,10 @@ fn test_list_owned_txouts() {
         assert_eq!(
             balance,
             Balance {
-                immature: 0,              // coinbase matured
-                trusted_pending: 15000,   // tx5
-                untrusted_pending: 20000, // tx4
-                confirmed: 80000          // tx1 + tx3
+                immature: Amount::ZERO,                     // coinbase matured
+                trusted_pending: Amount::from_sat(15000),   // tx5
+                untrusted_pending: Amount::from_sat(20000), // tx4
+                confirmed: Amount::from_sat(80000)          // tx1 + tx3
             }
         );
     }

crates/chain/tests/test_keychain_txout_index.rs

@@ -4,152 +4,266 @@
 mod common;
 use bdk_chain::{
     collections::BTreeMap,
-    keychain::{DerivationAdditions, KeychainTxOutIndex},
+    indexed_tx_graph::Indexer,
+    keychain::{self, ChangeSet, KeychainTxOutIndex},
+    Append, DescriptorExt, DescriptorId,
 };
 
-use bitcoin::{secp256k1::Secp256k1, OutPoint, Script, Transaction, TxOut};
+use bitcoin::{secp256k1::Secp256k1, Amount, OutPoint, ScriptBuf, Transaction, TxOut};
 use miniscript::{Descriptor, DescriptorPublicKey};
 
+use crate::common::DESCRIPTORS;
+
 #[derive(Clone, Debug, PartialEq, Eq, Ord, PartialOrd)]
 enum TestKeychain {
     External,
     Internal,
 }
 
-fn init_txout_index() -> (
-    bdk_chain::keychain::KeychainTxOutIndex<TestKeychain>,
-    Descriptor<DescriptorPublicKey>,
-    Descriptor<DescriptorPublicKey>,
-) {
-    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::default();
-
+fn parse_descriptor(descriptor: &str) -> Descriptor<DescriptorPublicKey> {
     let secp = bdk_chain::bitcoin::secp256k1::Secp256k1::signing_only();
-    let (external_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/0/*)").unwrap();
-    let (internal_descriptor,_) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "tr([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/*)").unwrap();
-
-    txout_index.add_keychain(TestKeychain::External, external_descriptor.clone());
-    txout_index.add_keychain(TestKeychain::Internal, internal_descriptor.clone());
-
-    (txout_index, external_descriptor, internal_descriptor)
+    Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, descriptor)
+        .unwrap()
+        .0
 }
 
-fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> Script {
+fn init_txout_index(
+    external_descriptor: Descriptor<DescriptorPublicKey>,
+    internal_descriptor: Descriptor<DescriptorPublicKey>,
+    lookahead: u32,
+) -> bdk_chain::keychain::KeychainTxOutIndex<TestKeychain> {
+    let mut txout_index = bdk_chain::keychain::KeychainTxOutIndex::<TestKeychain>::new(lookahead);
+
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::External, external_descriptor)
+        .unwrap();
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::Internal, internal_descriptor)
+        .unwrap();
+
+    txout_index
+}
+
+fn spk_at_index(descriptor: &Descriptor<DescriptorPublicKey>, index: u32) -> ScriptBuf {
     descriptor
         .derived_descriptor(&Secp256k1::verification_only(), index)
         .expect("must derive")
         .script_pubkey()
 }
 
+// We create two empty changesets lhs and rhs, we then insert various descriptors with various
+// last_revealed, append rhs to lhs, and check that the result is consistent with these rules:
+// - Existing index doesn't update if the new index in `other` is lower than `self`.
+// - Existing index updates if the new index in `other` is higher than `self`.
+// - Existing index is unchanged if keychain doesn't exist in `other`.
+// - New keychain gets added if the keychain is in `other` but not in `self`.
 #[test]
-fn test_set_all_derivation_indices() {
-    let (mut txout_index, _, _) = init_txout_index();
-    let derive_to: BTreeMap<_, _> =
-        [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into();
+fn append_changesets_check_last_revealed() {
+    let secp = bitcoin::secp256k1::Secp256k1::signing_only();
+    let descriptor_ids: Vec<_> = DESCRIPTORS
+        .iter()
+        .take(4)
+        .map(|d| {
+            Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, d)
+                .unwrap()
+                .0
+                .descriptor_id()
+        })
+        .collect();
+
+    let mut lhs_di = BTreeMap::<DescriptorId, u32>::default();
+    let mut rhs_di = BTreeMap::<DescriptorId, u32>::default();
+    lhs_di.insert(descriptor_ids[0], 7);
+    lhs_di.insert(descriptor_ids[1], 0);
+    lhs_di.insert(descriptor_ids[2], 3);
+
+    rhs_di.insert(descriptor_ids[0], 3); // value less than lhs desc 0
+    rhs_di.insert(descriptor_ids[1], 5); // value more than lhs desc 1
+    lhs_di.insert(descriptor_ids[3], 4); // key doesn't exist in lhs
+
+    let mut lhs = ChangeSet {
+        keychains_added: BTreeMap::<(), _>::new(),
+        last_revealed: lhs_di,
+    };
+    let rhs = ChangeSet {
+        keychains_added: BTreeMap::<(), _>::new(),
+        last_revealed: rhs_di,
+    };
+    lhs.append(rhs);
+
+    // Existing index doesn't update if the new index in `other` is lower than `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[0]), Some(&7));
+    // Existing index updates if the new index in `other` is higher than `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[1]), Some(&5));
+    // Existing index is unchanged if keychain doesn't exist in `other`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[2]), Some(&3));
+    // New keychain gets added if the keychain is in `other` but not in `self`.
+    assert_eq!(lhs.last_revealed.get(&descriptor_ids[3]), Some(&4));
+}
+
+#[test]
+fn when_apply_contradictory_changesets_they_are_ignored() {
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0);
     assert_eq!(
-        txout_index.reveal_to_target_multi(&derive_to).1.as_inner(),
-        &derive_to
+        txout_index.keychains().collect::<Vec<_>>(),
+        vec![
+            (&TestKeychain::External, &external_descriptor),
+            (&TestKeychain::Internal, &internal_descriptor)
+        ]
     );
-    assert_eq!(txout_index.last_revealed_indices(), &derive_to);
+
+    let changeset = ChangeSet {
+        keychains_added: [(TestKeychain::External, internal_descriptor.clone())].into(),
+        last_revealed: [].into(),
+    };
+    txout_index.apply_changeset(changeset);
+
     assert_eq!(
-        txout_index.reveal_to_target_multi(&derive_to).1,
-        DerivationAdditions::default(),
-        "no changes if we set to the same thing"
+        txout_index.keychains().collect::<Vec<_>>(),
+        vec![
+            (&TestKeychain::External, &external_descriptor),
+            (&TestKeychain::Internal, &internal_descriptor)
+        ]
+    );
+
+    let changeset = ChangeSet {
+        keychains_added: [(TestKeychain::Internal, external_descriptor.clone())].into(),
+        last_revealed: [].into(),
+    };
+    txout_index.apply_changeset(changeset);
+
+    assert_eq!(
+        txout_index.keychains().collect::<Vec<_>>(),
+        vec![
+            (&TestKeychain::External, &external_descriptor),
+            (&TestKeychain::Internal, &internal_descriptor)
+        ]
     );
 }
 
+#[test]
+fn test_set_all_derivation_indices() {
+    use bdk_chain::indexed_tx_graph::Indexer;
+
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0);
+    let derive_to: BTreeMap<_, _> =
+        [(TestKeychain::External, 12), (TestKeychain::Internal, 24)].into();
+    let last_revealed: BTreeMap<_, _> = [
+        (external_descriptor.descriptor_id(), 12),
+        (internal_descriptor.descriptor_id(), 24),
+    ]
+    .into();
+    assert_eq!(
+        txout_index.reveal_to_target_multi(&derive_to),
+        ChangeSet {
+            keychains_added: BTreeMap::new(),
+            last_revealed: last_revealed.clone()
+        }
+    );
+    assert_eq!(txout_index.last_revealed_indices(), derive_to);
+    assert_eq!(
+        txout_index.reveal_to_target_multi(&derive_to),
+        keychain::ChangeSet::default(),
+        "no changes if we set to the same thing"
+    );
+    assert_eq!(txout_index.initial_changeset().last_revealed, last_revealed);
+}
+
 #[test]
 fn test_lookahead() {
-    let (mut txout_index, external_desc, internal_desc) = init_txout_index();
-
-    // ensure it does not break anything if lookahead is set multiple times
-    (0..=10).for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::External, lookahead));
-    (0..=20)
-        .filter(|v| v % 2 == 0)
-        .for_each(|lookahead| txout_index.set_lookahead(&TestKeychain::Internal, lookahead));
-
-    assert_eq!(txout_index.inner().all_spks().len(), 30);
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 10);
 
     // given:
     // - external lookahead set to 10
-    // - internal lookahead set to 20
     // when:
     // - set external derivation index to value higher than last, but within the lookahead value
     // expect:
     // - scripts cached in spk_txout_index should increase correctly
     // - stored scripts of external keychain should be of expected counts
     for index in (0..20).skip_while(|i| i % 2 == 1) {
-        let (revealed_spks, revealed_additions) =
-            txout_index.reveal_to_target(&TestKeychain::External, index);
+        let (revealed_spks, revealed_changeset) = txout_index
+            .reveal_to_target(&TestKeychain::External, index)
+            .unwrap();
         assert_eq!(
-            revealed_spks.collect::<Vec<_>>(),
-            vec![(index, spk_at_index(&external_desc, index))],
+            revealed_spks,
+            vec![(index, spk_at_index(&external_descriptor, index))],
         );
         assert_eq!(
-            revealed_additions.as_inner(),
-            &[(TestKeychain::External, index)].into()
+            &revealed_changeset.last_revealed,
+            &[(external_descriptor.descriptor_id(), index)].into()
         );
 
         assert_eq!(
             txout_index.inner().all_spks().len(),
             10 /* external lookahead */ +
-            20 /* internal lookahead */ +
+            10 /* internal lookahead */ +
             index as usize + 1 /* `derived` count */
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::External)
+                .unused_keychain_spks(&TestKeychain::External)
                 .count(),
             index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .unused_spks_of_keychain(&TestKeychain::Internal)
+                .unused_keychain_spks(&TestKeychain::Internal)
                 .count(),
             0,
         );
     }
 
     // given:
-    // - internal lookahead is 20
+    // - internal lookahead is 10
     // - internal derivation index is `None`
     // when:
     // - derivation index is set ahead of current derivation index + lookahead
     // expect:
     // - scripts cached in spk_txout_index should increase correctly, a.k.a. no scripts are skipped
-    let (revealed_spks, revealed_additions) =
-        txout_index.reveal_to_target(&TestKeychain::Internal, 24);
+    let (revealed_spks, revealed_changeset) = txout_index
+        .reveal_to_target(&TestKeychain::Internal, 24)
+        .unwrap();
     assert_eq!(
-        revealed_spks.collect::<Vec<_>>(),
+        revealed_spks,
         (0..=24)
-            .map(|index| (index, spk_at_index(&internal_desc, index)))
+            .map(|index| (index, spk_at_index(&internal_descriptor, index)))
             .collect::<Vec<_>>(),
     );
     assert_eq!(
-        revealed_additions.as_inner(),
-        &[(TestKeychain::Internal, 24)].into()
+        &revealed_changeset.last_revealed,
+        &[(internal_descriptor.descriptor_id(), 24)].into()
     );
     assert_eq!(
         txout_index.inner().all_spks().len(),
         10 /* external lookahead */ +
-        20 /* internal lookahead */ +
+        10 /* internal lookahead */ +
         20 /* external stored index count */ +
         25 /* internal stored index count */
     );
     assert_eq!(
         txout_index
-            .revealed_spks_of_keychain(&TestKeychain::Internal)
+            .revealed_keychain_spks(&TestKeychain::Internal)
             .count(),
         25,
     );
@@ -174,21 +288,23 @@ fn test_lookahead() {
         let tx = Transaction {
             output: vec![
                 TxOut {
-                    script_pubkey: external_desc
+                    script_pubkey: external_descriptor
                         .at_derivation_index(external_index)
+                        .unwrap()
                         .script_pubkey(),
-                    value: 10_000,
+                    value: Amount::from_sat(10_000),
                 },
                 TxOut {
-                    script_pubkey: internal_desc
+                    script_pubkey: internal_descriptor
                         .at_derivation_index(internal_index)
+                        .unwrap()
                         .script_pubkey(),
-                    value: 10_000,
+                    value: Amount::from_sat(10_000),
                 },
             ],
             ..common::new_tx(external_index)
         };
-        assert_eq!(txout_index.scan(&tx), DerivationAdditions::default());
+        assert_eq!(txout_index.index_tx(&tx), keychain::ChangeSet::default());
         assert_eq!(
             txout_index.last_revealed_index(&TestKeychain::External),
             Some(last_external_index)
@@ -199,13 +315,13 @@ fn test_lookahead() {
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::External)
+                .revealed_keychain_spks(&TestKeychain::External)
                 .count(),
             last_external_index as usize + 1,
         );
         assert_eq!(
             txout_index
-                .revealed_spks_of_keychain(&TestKeychain::Internal)
+                .revealed_keychain_spks(&TestKeychain::Internal)
                 .count(),
             last_internal_index as usize + 1,
         );
@@ -219,25 +335,35 @@ fn test_lookahead() {
 // - last used index should change as expected
 #[test]
 fn test_scan_with_lookahead() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
-    txout_index.set_lookahead_for_all(10);
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index =
+        init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 10);
 
-    let spks: BTreeMap<u32, Script> = [0, 10, 20, 30]
+    let spks: BTreeMap<u32, ScriptBuf> = [0, 10, 20, 30]
         .into_iter()
-        .map(|i| (i, external_desc.at_derivation_index(i).script_pubkey()))
+        .map(|i| {
+            (
+                i,
+                external_descriptor
+                    .at_derivation_index(i)
+                    .unwrap()
+                    .script_pubkey(),
+            )
+        })
         .collect();
 
     for (&spk_i, spk) in &spks {
         let op = OutPoint::new(h!("fake tx"), spk_i);
         let txout = TxOut {
             script_pubkey: spk.clone(),
-            value: 0,
+            value: Amount::ZERO,
         };
 
-        let additions = txout_index.scan_txout(op, &txout);
+        let changeset = txout_index.index_txout(op, &txout);
         assert_eq!(
-            additions.as_inner(),
-            &[(TestKeychain::External, spk_i)].into()
+            &changeset.last_revealed,
+            &[(external_descriptor.descriptor_id(), spk_i)].into()
         );
         assert_eq!(
             txout_index.last_revealed_index(&TestKeychain::External),
@@ -250,37 +376,43 @@ fn test_scan_with_lookahead() {
     }
 
     // now try with index 41 (lookahead surpassed), we expect that the txout to not be indexed
-    let spk_41 = external_desc.at_derivation_index(41).script_pubkey();
+    let spk_41 = external_descriptor
+        .at_derivation_index(41)
+        .unwrap()
+        .script_pubkey();
     let op = OutPoint::new(h!("fake tx"), 41);
     let txout = TxOut {
         script_pubkey: spk_41,
-        value: 0,
+        value: Amount::ZERO,
     };
-    let additions = txout_index.scan_txout(op, &txout);
-    assert!(additions.is_empty());
+    let changeset = txout_index.index_txout(op, &txout);
+    assert!(changeset.is_empty());
 }
 
 #[test]
+#[rustfmt::skip]
 fn test_wildcard_derivations() {
-    let (mut txout_index, external_desc, _) = init_txout_index();
-    let external_spk_0 = external_desc.at_derivation_index(0).script_pubkey();
-    let external_spk_16 = external_desc.at_derivation_index(16).script_pubkey();
-    let external_spk_26 = external_desc.at_derivation_index(26).script_pubkey();
-    let external_spk_27 = external_desc.at_derivation_index(27).script_pubkey();
+    let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+    let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+    let mut txout_index = init_txout_index(external_descriptor.clone(), internal_descriptor.clone(), 0);
+    let external_spk_0 = external_descriptor.at_derivation_index(0).unwrap().script_pubkey();
+    let external_spk_16 = external_descriptor.at_derivation_index(16).unwrap().script_pubkey();
+    let external_spk_26 = external_descriptor.at_derivation_index(26).unwrap().script_pubkey();
+    let external_spk_27 = external_descriptor.at_derivation_index(27).unwrap().script_pubkey();
 
     // - nothing is derived
     // - unused list is also empty
     //
     // - next_derivation_index() == (0, true)
-    // - derive_new() == ((0, <spk>), DerivationAdditions)
-    // - next_unused() == ((0, <spk>), DerivationAdditions:is_empty())
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, &external_spk_0));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0_u32, &external_spk_0));
-    assert_eq!(changeset.as_inner(), &[].into());
+    // - derive_new() == ((0, <spk>), keychain::ChangeSet)
+    // - next_unused() == ((0, <spk>), keychain::ChangeSet:is_empty())
+    assert_eq!(txout_index.next_index(&TestKeychain::External).unwrap(), (0, true));
+    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External).unwrap();
+    assert_eq!(spk, (0_u32, external_spk_0.clone()));
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 0)].into());
+    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External).unwrap();
+    assert_eq!(spk, (0_u32, external_spk_0.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // - derived till 25
     // - used all spks till 15.
@@ -288,47 +420,51 @@ fn test_wildcard_derivations() {
     // - unused list: [16, 18, 19, 21, 22, 24, 25]
 
     // - next_derivation_index() = (26, true)
-    // - derive_new() = ((26, <spk>), DerivationAdditions)
-    // - next_unused() == ((16, <spk>), DerivationAdditions::is_empty())
+    // - derive_new() = ((26, <spk>), keychain::ChangeSet)
+    // - next_unused() == ((16, <spk>), keychain::ChangeSet::is_empty())
     let _ = txout_index.reveal_to_target(&TestKeychain::External, 25);
 
     (0..=15)
-        .chain(vec![17, 20, 23].into_iter())
-        .for_each(|index| assert!(txout_index.mark_used(&TestKeychain::External, index)));
+        .chain([17, 20, 23])
+        .for_each(|index| assert!(txout_index.mark_used(TestKeychain::External, index)));
 
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (26, true));
+    assert_eq!(txout_index.next_index(&TestKeychain::External).unwrap(), (26, true));
 
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (26, &external_spk_26));
+    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External).unwrap();
+    assert_eq!(spk, (26, external_spk_26));
 
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 26)].into());
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 26)].into());
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (16, &external_spk_16));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External).unwrap();
+    assert_eq!(spk, (16, external_spk_16));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // - Use all the derived till 26.
-    // - next_unused() = ((27, <spk>), DerivationAdditions)
+    // - next_unused() = ((27, <spk>), keychain::ChangeSet)
     (0..=26).for_each(|index| {
-        txout_index.mark_used(&TestKeychain::External, index);
+        txout_index.mark_used(TestKeychain::External, index);
     });
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (27, &external_spk_27));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 27)].into());
+    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External).unwrap();
+    assert_eq!(spk, (27, external_spk_27));
+    assert_eq!(&changeset.last_revealed, &[(external_descriptor.descriptor_id(), 27)].into());
 }
 
 #[test]
 fn test_non_wildcard_derivations() {
-    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::default();
+    let mut txout_index = KeychainTxOutIndex::<TestKeychain>::new(0);
 
     let secp = bitcoin::secp256k1::Secp256k1::signing_only();
-    let (no_wildcard_descriptor, _) = Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, "wpkh([73c5da0a/86'/0'/0']xprv9xgqHN7yz9MwCkxsBPN5qetuNdQSUttZNKw1dcYTV4mkaAFiBVGQziHs3NRSWMkCzvgjEe3n9xV8oYywvM8at9yRqyaZVz6TYYhX98VjsUk/1/0)").unwrap();
+    let (no_wildcard_descriptor, _) =
+        Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, DESCRIPTORS[6]).unwrap();
     let external_spk = no_wildcard_descriptor
         .at_derivation_index(0)
+        .unwrap()
         .script_pubkey();
 
-    txout_index.add_keychain(TestKeychain::External, no_wildcard_descriptor);
+    let _ = txout_index
+        .insert_descriptor(TestKeychain::External, no_wildcard_descriptor.clone())
+        .unwrap();
 
     // given:
     // - `txout_index` with no stored scripts
@@ -336,33 +472,292 @@ fn test_non_wildcard_derivations() {
     // - next derivation index should be new
     // - when we derive a new script, script @ index 0
     // - when we get the next unused script, script @ index 0
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, true));
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
-    assert_eq!(changeset.as_inner(), &[(TestKeychain::External, 0)].into());
+    assert_eq!(
+        txout_index.next_index(&TestKeychain::External).unwrap(),
+        (0, true)
+    );
+    let (spk, changeset) = txout_index
+        .reveal_next_spk(&TestKeychain::External)
+        .unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(
+        &changeset.last_revealed,
+        &[(no_wildcard_descriptor.descriptor_id(), 0)].into()
+    );
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index
+        .next_unused_spk(&TestKeychain::External)
+        .unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
     // given:
     // - the non-wildcard descriptor already has a stored and used script
     // expect:
     // - next derivation index should not be new
     // - derive new and next unused should return the old script
-    // - store_up_to should not panic and return empty additions
-    assert_eq!(txout_index.next_index(&TestKeychain::External), (0, false));
-    txout_index.mark_used(&TestKeychain::External, 0);
+    // - store_up_to should not panic and return empty changeset
+    assert_eq!(
+        txout_index.next_index(&TestKeychain::External).unwrap(),
+        (0, false)
+    );
+    txout_index.mark_used(TestKeychain::External, 0);
 
-    let (spk, changeset) = txout_index.reveal_next_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
-    assert_eq!(changeset.as_inner(), &[].into());
+    let (spk, changeset) = txout_index
+        .reveal_next_spk(&TestKeychain::External)
+        .unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
 
-    let (spk, changeset) = txout_index.next_unused_spk(&TestKeychain::External);
-    assert_eq!(spk, (0, &external_spk));
-    assert_eq!(changeset.as_inner(), &[].into());
-    let (revealed_spks, revealed_additions) =
-        txout_index.reveal_to_target(&TestKeychain::External, 200);
-    assert_eq!(revealed_spks.count(), 0);
-    assert!(revealed_additions.is_empty());
+    let (spk, changeset) = txout_index
+        .next_unused_spk(&TestKeychain::External)
+        .unwrap();
+    assert_eq!(spk, (0, external_spk.clone()));
+    assert_eq!(&changeset.last_revealed, &[].into());
+    let (revealed_spks, revealed_changeset) = txout_index
+        .reveal_to_target(&TestKeychain::External, 200)
+        .unwrap();
+    assert_eq!(revealed_spks.len(), 0);
+    assert!(revealed_changeset.is_empty());
+
+    // we check that spks_of_keychain returns a SpkIterator with just one element
+    assert_eq!(
+        txout_index
+            .revealed_keychain_spks(&TestKeychain::External)
+            .count(),
+        1,
+    );
+}
+
+/// Check that calling `lookahead_to_target` stores the expected spks.
+#[test]
+fn lookahead_to_target() {
+    #[derive(Default)]
+    struct TestCase {
+        /// Global lookahead value.
+        lookahead: u32,
+        /// Last revealed index for external keychain.
+        external_last_revealed: Option<u32>,
+        /// Last revealed index for internal keychain.
+        internal_last_revealed: Option<u32>,
+        /// Call `lookahead_to_target(External, u32)`.
+        external_target: Option<u32>,
+        /// Call `lookahead_to_target(Internal, u32)`.
+        internal_target: Option<u32>,
+    }
+
+    let test_cases = &[
+        TestCase {
+            lookahead: 0,
+            external_target: Some(100),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 10,
+            internal_target: Some(99),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 100,
+            internal_target: Some(9),
+            external_target: Some(10),
+            ..Default::default()
+        },
+        TestCase {
+            lookahead: 12,
+            external_last_revealed: Some(2),
+            internal_last_revealed: Some(2),
+            internal_target: Some(15),
+            external_target: Some(13),
+        },
+        TestCase {
+            lookahead: 13,
+            external_last_revealed: Some(100),
+            internal_last_revealed: Some(21),
+            internal_target: Some(120),
+            external_target: Some(130),
+        },
+    ];
+
+    for t in test_cases {
+        let external_descriptor = parse_descriptor(DESCRIPTORS[0]);
+        let internal_descriptor = parse_descriptor(DESCRIPTORS[1]);
+        let mut index = init_txout_index(
+            external_descriptor.clone(),
+            internal_descriptor.clone(),
+            t.lookahead,
+        );
+
+        if let Some(last_revealed) = t.external_last_revealed {
+            let _ = index.reveal_to_target(&TestKeychain::External, last_revealed);
+        }
+        if let Some(last_revealed) = t.internal_last_revealed {
+            let _ = index.reveal_to_target(&TestKeychain::Internal, last_revealed);
+        }
+
+        let keychain_test_cases = [
+            (
+                TestKeychain::External,
+                t.external_last_revealed,
+                t.external_target,
+            ),
+            (
+                TestKeychain::Internal,
+                t.internal_last_revealed,
+                t.internal_target,
+            ),
+        ];
+        for (keychain, last_revealed, target) in keychain_test_cases {
+            if let Some(target) = target {
+                let original_last_stored_index = match last_revealed {
+                    Some(last_revealed) => Some(last_revealed + t.lookahead),
+                    None => t.lookahead.checked_sub(1),
+                };
+                let exp_last_stored_index = match original_last_stored_index {
+                    Some(original_last_stored_index) => {
+                        Ord::max(target, original_last_stored_index)
+                    }
+                    None => target,
+                };
+                index.lookahead_to_target(&keychain, target);
+                let keys = index
+                    .inner()
+                    .all_spks()
+                    .range((keychain.clone(), 0)..=(keychain.clone(), u32::MAX))
+                    .map(|(k, _)| k.clone())
+                    .collect::<Vec<_>>();
+                let exp_keys = core::iter::repeat(keychain)
+                    .zip(0_u32..=exp_last_stored_index)
+                    .collect::<Vec<_>>();
+                assert_eq!(keys, exp_keys);
+            }
+        }
+    }
+}
+
+#[test]
+fn insert_descriptor_no_change() {
+    let secp = Secp256k1::signing_only();
+    let (desc, _) =
+        Descriptor::<DescriptorPublicKey>::parse_descriptor(&secp, DESCRIPTORS[0]).unwrap();
+    let mut txout_index = KeychainTxOutIndex::<()>::default();
+    assert_eq!(
+        txout_index.insert_descriptor((), desc.clone()),
+        Ok(keychain::ChangeSet {
+            keychains_added: [((), desc.clone())].into(),
+            last_revealed: Default::default()
+        }),
+    );
+    assert_eq!(
+        txout_index.insert_descriptor((), desc.clone()),
+        Ok(keychain::ChangeSet::default()),
+        "inserting the same descriptor for keychain should return an empty changeset",
+    );
+}
+
+#[test]
+#[cfg(not(debug_assertions))]
+fn applying_changesets_one_by_one_vs_aggregate_must_have_same_result() {
+    let desc = parse_descriptor(DESCRIPTORS[0]);
+    let changesets: &[ChangeSet<TestKeychain>] = &[
+        ChangeSet {
+            keychains_added: [(TestKeychain::Internal, desc.clone())].into(),
+            last_revealed: [].into(),
+        },
+        ChangeSet {
+            keychains_added: [(TestKeychain::External, desc.clone())].into(),
+            last_revealed: [(desc.descriptor_id(), 12)].into(),
+        },
+    ];
+
+    let mut indexer_a = KeychainTxOutIndex::<TestKeychain>::new(0);
+    for changeset in changesets {
+        indexer_a.apply_changeset(changeset.clone());
+    }
+
+    let mut indexer_b = KeychainTxOutIndex::<TestKeychain>::new(0);
+    let aggregate_changesets = changesets
+        .iter()
+        .cloned()
+        .reduce(|mut agg, cs| {
+            agg.append(cs);
+            agg
+        })
+        .expect("must aggregate changesets");
+    indexer_b.apply_changeset(aggregate_changesets);
+
+    assert_eq!(
+        indexer_a.keychains().collect::<Vec<_>>(),
+        indexer_b.keychains().collect::<Vec<_>>()
+    );
+    assert_eq!(
+        indexer_a.spk_at_index(TestKeychain::External, 0),
+        indexer_b.spk_at_index(TestKeychain::External, 0)
+    );
+    assert_eq!(
+        indexer_a.spk_at_index(TestKeychain::Internal, 0),
+        indexer_b.spk_at_index(TestKeychain::Internal, 0)
+    );
+    assert_eq!(
+        indexer_a.last_revealed_indices(),
+        indexer_b.last_revealed_indices()
+    );
+}
+
+#[test]
+fn assigning_same_descriptor_to_multiple_keychains_should_error() {
+    let desc = parse_descriptor(DESCRIPTORS[0]);
+    let mut indexer = KeychainTxOutIndex::<TestKeychain>::new(0);
+    let _ = indexer
+        .insert_descriptor(TestKeychain::Internal, desc.clone())
+        .unwrap();
+    assert!(indexer
+        .insert_descriptor(TestKeychain::External, desc)
+        .is_err())
+}
+
+#[test]
+fn reassigning_keychain_to_a_new_descriptor_should_error() {
+    let desc1 = parse_descriptor(DESCRIPTORS[0]);
+    let desc2 = parse_descriptor(DESCRIPTORS[1]);
+    let mut indexer = KeychainTxOutIndex::<TestKeychain>::new(0);
+    let _ = indexer.insert_descriptor(TestKeychain::Internal, desc1);
+    assert!(indexer
+        .insert_descriptor(TestKeychain::Internal, desc2)
+        .is_err());
+}
+
+#[test]
+fn when_querying_over_a_range_of_keychains_the_utxos_should_show_up() {
+    let mut indexer = KeychainTxOutIndex::<usize>::new(0);
+    let mut tx = common::new_tx(0);
+
+    for (i, descriptor) in DESCRIPTORS.iter().enumerate() {
+        let descriptor = parse_descriptor(descriptor);
+        let _ = indexer.insert_descriptor(i, descriptor.clone()).unwrap();
+        if i != 4 {
+            // skip one in the middle to see if uncovers any bugs
+            indexer.reveal_next_spk(&i);
+        }
+        tx.output.push(TxOut {
+            script_pubkey: descriptor.at_derivation_index(0).unwrap().script_pubkey(),
+            value: Amount::from_sat(10_000),
+        });
+    }
+
+    let n_spks = DESCRIPTORS.len() - /*we skipped one*/ 1;
+
+    let _ = indexer.index_tx(&tx);
+    assert_eq!(indexer.outpoints().len(), n_spks);
+
+    assert_eq!(indexer.revealed_spks(0..DESCRIPTORS.len()).count(), n_spks);
+    assert_eq!(indexer.revealed_spks(1..4).count(), 4 - 1);
+    assert_eq!(
+        indexer.net_value(&tx, 0..DESCRIPTORS.len()).to_sat(),
+        (10_000 * n_spks) as i64
+    );
+    assert_eq!(
+        indexer.net_value(&tx, 3..6).to_sat(),
+        (10_000 * (6 - 3 - /*the skipped one*/ 1)) as i64
+    );
 }

crates/chain/tests/test_spk_txout_index.rs

@@ -1,64 +1,88 @@
-use bdk_chain::SpkTxOutIndex;
-use bitcoin::{hashes::hex::FromHex, OutPoint, PackedLockTime, Script, Transaction, TxIn, TxOut};
+use bdk_chain::{indexed_tx_graph::Indexer, SpkTxOutIndex};
+use bitcoin::{
+    absolute, transaction, Amount, OutPoint, ScriptBuf, SignedAmount, Transaction, TxIn, TxOut,
+};
 
 #[test]
 fn spk_txout_sent_and_received() {
-    let spk1 = Script::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
-    let spk2 = Script::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
+    let spk1 = ScriptBuf::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
+    let spk2 = ScriptBuf::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
 
     let mut index = SpkTxOutIndex::default();
     index.insert_spk(0, spk1.clone());
     index.insert_spk(1, spk2.clone());
 
     let tx1 = Transaction {
-        version: 0x02,
-        lock_time: PackedLockTime(0),
+        version: transaction::Version::TWO,
+        lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
-            value: 42_000,
+            value: Amount::from_sat(42_000),
             script_pubkey: spk1.clone(),
         }],
     };
 
-    assert_eq!(index.sent_and_received(&tx1), (0, 42_000));
-    assert_eq!(index.net_value(&tx1), 42_000);
-    index.scan(&tx1);
     assert_eq!(
-        index.sent_and_received(&tx1),
-        (0, 42_000),
+        index.sent_and_received(&tx1, ..),
+        (Amount::from_sat(0), Amount::from_sat(42_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx1, ..1),
+        (Amount::from_sat(0), Amount::from_sat(42_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx1, 1..),
+        (Amount::from_sat(0), Amount::from_sat(0))
+    );
+    assert_eq!(index.net_value(&tx1, ..), SignedAmount::from_sat(42_000));
+    index.index_tx(&tx1);
+    assert_eq!(
+        index.sent_and_received(&tx1, ..),
+        (Amount::from_sat(0), Amount::from_sat(42_000)),
         "shouldn't change after scanning"
     );
 
     let tx2 = Transaction {
-        version: 0x1,
-        lock_time: PackedLockTime(0),
+        version: transaction::Version::ONE,
+        lock_time: absolute::LockTime::ZERO,
         input: vec![TxIn {
             previous_output: OutPoint {
-                txid: tx1.txid(),
+                txid: tx1.compute_txid(),
                 vout: 0,
             },
             ..Default::default()
         }],
         output: vec![
             TxOut {
-                value: 20_000,
+                value: Amount::from_sat(20_000),
                 script_pubkey: spk2,
             },
             TxOut {
                 script_pubkey: spk1,
-                value: 30_000,
+                value: Amount::from_sat(30_000),
             },
         ],
     };
 
-    assert_eq!(index.sent_and_received(&tx2), (42_000, 50_000));
-    assert_eq!(index.net_value(&tx2), 8_000);
+    assert_eq!(
+        index.sent_and_received(&tx2, ..),
+        (Amount::from_sat(42_000), Amount::from_sat(50_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx2, ..1),
+        (Amount::from_sat(42_000), Amount::from_sat(30_000))
+    );
+    assert_eq!(
+        index.sent_and_received(&tx2, 1..),
+        (Amount::from_sat(0), Amount::from_sat(20_000))
+    );
+    assert_eq!(index.net_value(&tx2, ..), SignedAmount::from_sat(8_000));
 }
 
 #[test]
 fn mark_used() {
-    let spk1 = Script::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
-    let spk2 = Script::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
+    let spk1 = ScriptBuf::from_hex("001404f1e52ce2bab3423c6a8c63b7cd730d8f12542c").unwrap();
+    let spk2 = ScriptBuf::from_hex("00142b57404ae14f08c3a0c903feb2af7830605eb00f").unwrap();
 
     let mut spk_index = SpkTxOutIndex::default();
     spk_index.insert_spk(1, spk1.clone());
@@ -73,16 +97,16 @@ fn mark_used() {
     assert!(spk_index.is_used(&1));
 
     let tx1 = Transaction {
-        version: 0x02,
-        lock_time: PackedLockTime(0),
+        version: transaction::Version::TWO,
+        lock_time: absolute::LockTime::ZERO,
         input: vec![],
         output: vec![TxOut {
-            value: 42_000,
+            value: Amount::from_sat(42_000),
             script_pubkey: spk1,
         }],
     };
 
-    spk_index.scan(&tx1);
+    spk_index.index_tx(&tx1);
     spk_index.unmark_used(&1);
     assert!(
         spk_index.is_used(&1),

crates/chain/tests/test_tx_graph_conflicts.rs

@@ -0,0 +1,670 @@
+#![cfg(feature = "miniscript")]
+
+#[macro_use]
+mod common;
+
+use std::collections::{BTreeSet, HashSet};
+
+use bdk_chain::{keychain::Balance, BlockId};
+use bitcoin::{Amount, OutPoint, Script};
+use common::*;
+
+#[allow(dead_code)]
+struct Scenario<'a> {
+    /// Name of the test scenario
+    name: &'a str,
+    /// Transaction templates
+    tx_templates: &'a [TxTemplate<'a, BlockId>],
+    /// Names of txs that must exist in the output of `list_chain_txs`
+    exp_chain_txs: HashSet<&'a str>,
+    /// Outpoints that must exist in the output of `filter_chain_txouts`
+    exp_chain_txouts: HashSet<(&'a str, u32)>,
+    /// Outpoints of UTXOs that must exist in the output of `filter_chain_unspents`
+    exp_unspents: HashSet<(&'a str, u32)>,
+    /// Expected balances
+    exp_balance: Balance,
+}
+
+/// This test ensures that [`TxGraph`] will reliably filter out irrelevant transactions when
+/// presented with multiple conflicting transaction scenarios using the [`TxTemplate`] structure.
+/// This test also checks that [`TxGraph::list_chain_txs`], [`TxGraph::filter_chain_txouts`],
+/// [`TxGraph::filter_chain_unspents`], and [`TxGraph::balance`] return correct data.
+#[test]
+fn test_tx_conflict_handling() {
+    // Create Local chains
+    let local_chain = local_chain!(
+        (0, h!("A")),
+        (1, h!("B")),
+        (2, h!("C")),
+        (3, h!("D")),
+        (4, h!("E")),
+        (5, h!("F")),
+        (6, h!("G"))
+    );
+    let chain_tip = local_chain.tip().block_id();
+
+    let scenarios = [
+        Scenario {
+            name: "coinbase tx cannot be in mempool and be unconfirmed",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "unconfirmed_coinbase",
+                    inputs: &[TxInTemplate::Coinbase],
+                    outputs: &[TxOutTemplate::new(5000, Some(0))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "confirmed_genesis",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(1))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "unconfirmed_conflict",
+                    inputs: &[
+                        TxInTemplate::PrevTx("confirmed_genesis", 0), 
+                        TxInTemplate::PrevTx("unconfirmed_coinbase", 0)
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "confirmed_conflict",
+                    inputs: &[TxInTemplate::PrevTx("confirmed_genesis", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(3))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["confirmed_genesis", "confirmed_conflict"]),
+            exp_chain_txouts: HashSet::from([("confirmed_genesis", 0), ("confirmed_conflict", 0)]),
+            exp_unspents: HashSet::from([("confirmed_conflict", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(20000),
+            },
+        },
+        Scenario {
+            name: "2 unconfirmed txs with same last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    outputs: &[TxOutTemplate::new(40000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+            ],
+            // the txgraph is going to pick tx_conflict_2 because of higher lexicographical txid
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_2"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_2", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "2 unconfirmed txs with different last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0)), TxOutTemplate::new(10000, Some(1))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0),  TxInTemplate::PrevTx("tx1", 1)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_2"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx1", 1), ("tx_conflict_2", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_2", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "3 unconfirmed txs with different last_seens conflict",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_3",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(40000, Some(3))],
+                    last_seen: Some(400),
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_3"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_3", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_3", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(40000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "unconfirmed tx conflicts with tx in orphaned block, orphaned higher last_seen",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_orphaned_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    anchors: &[block_id!(4, "Orphaned Block")],
+                    last_seen: Some(300),
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_orphaned_conflict"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_orphaned_conflict", 0)]),
+            exp_unspents: HashSet::from([("tx_orphaned_conflict", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "unconfirmed tx conflicts with tx in orphaned block, orphaned lower last_seen",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_orphaned_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    anchors: &[block_id!(4, "Orphaned Block")],
+                    last_seen: Some(100),
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_conflict_1"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_conflict_1", 0)]),
+            exp_unspents: HashSet::from([("tx_conflict_1", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(20000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "multiple unconfirmed txs conflict with a confirmed tx",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "tx1",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_1",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_2",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_conflict_3",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(40000, Some(3))],
+                    last_seen: Some(400),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "tx_confirmed_conflict",
+                    inputs: &[TxInTemplate::PrevTx("tx1", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+            ],
+            exp_chain_txs: HashSet::from(["tx1", "tx_confirmed_conflict"]),
+            exp_chain_txouts: HashSet::from([("tx1", 0), ("tx_confirmed_conflict", 0)]),
+            exp_unspents: HashSet::from([("tx_confirmed_conflict", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B, all the transactions are unconfirmed, B' has higher last_seen than B",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    last_seen: Some(22),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(23),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    last_seen: Some(24),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    last_seen: Some(25),
+                    ..Default::default()
+                },
+            ],
+            // A, B, C will appear in the list methods
+            // This is because B' has a higher last seen than B, but C has a higher
+            // last seen than B', so B and C are considered canonical
+            exp_chain_txs: HashSet::from(["A", "B", "C"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B", 0), ("C", 0)]),
+            exp_unspents: HashSet::from([("C", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B, A and B' are in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // B and C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(20000),
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends B', A and B' are in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(2))],
+                    anchors: &[block_id!(4, "E")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[TxInTemplate::PrevTx("B'", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // B should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'", "C"]),
+            exp_chain_txouts: HashSet::from([
+                ("A", 0),
+                ("B'", 0),
+                ("C", 0),
+            ]),
+            exp_unspents: HashSet::from([("C", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, C spends both B and B', A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(30000, Some(2))],
+                    last_seen: Some(300),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(3))],
+                    ..Default::default()
+                },
+            ],
+            // C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::from_sat(30000),
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::ZERO,
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, B' is confirmed, C spends both B and B', A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(5))],
+                    ..Default::default()
+                },
+            ],
+            // C should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
+            },
+        },
+        Scenario {
+            name: "B and B' spend A and conflict, B' is confirmed, C spends both B and B', D spends C, A is in best chain",
+            tx_templates: &[
+                TxTemplate {
+                    tx_name: "A",
+                    inputs: &[TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(10000, Some(0))],
+                    anchors: &[block_id!(1, "B")],
+                    last_seen: None,
+                },
+                TxTemplate {
+                    tx_name: "B",
+                    inputs: &[TxInTemplate::PrevTx("A", 0), TxInTemplate::Bogus],
+                    outputs: &[TxOutTemplate::new(20000, Some(1))],
+                    last_seen: Some(200),
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "B'",
+                    inputs: &[TxInTemplate::PrevTx("A", 0)],
+                    outputs: &[TxOutTemplate::new(50000, Some(4))],
+                    anchors: &[block_id!(1, "B")],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "C",
+                    inputs: &[
+                        TxInTemplate::PrevTx("B", 0),
+                        TxInTemplate::PrevTx("B'", 0),
+                    ],
+                    outputs: &[TxOutTemplate::new(20000, Some(5))],
+                    ..Default::default()
+                },
+                TxTemplate {
+                    tx_name: "D",
+                    inputs: &[TxInTemplate::PrevTx("C", 0)],
+                    outputs: &[TxOutTemplate::new(20000, Some(6))],
+                    ..Default::default()
+                },
+            ],
+            // D should not appear in the list methods
+            exp_chain_txs: HashSet::from(["A", "B'"]),
+            exp_chain_txouts: HashSet::from([("A", 0), ("B'", 0)]),
+            exp_unspents: HashSet::from([("B'", 0)]),
+            exp_balance: Balance {
+                immature: Amount::ZERO,
+                trusted_pending: Amount::ZERO,
+                untrusted_pending: Amount::ZERO,
+                confirmed: Amount::from_sat(50000),
+            },
+        },
+    ];
+
+    for scenario in scenarios {
+        let (tx_graph, spk_index, exp_tx_ids) = init_graph(scenario.tx_templates.iter());
+
+        let txs = tx_graph
+            .list_chain_txs(&local_chain, chain_tip)
+            .map(|tx| tx.tx_node.txid)
+            .collect::<BTreeSet<_>>();
+        let exp_txs = scenario
+            .exp_chain_txs
+            .iter()
+            .map(|txid| *exp_tx_ids.get(txid).expect("txid must exist"))
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            txs, exp_txs,
+            "\n[{}] 'list_chain_txs' failed",
+            scenario.name
+        );
+
+        let txouts = tx_graph
+            .filter_chain_txouts(
+                &local_chain,
+                chain_tip,
+                spk_index.outpoints().iter().cloned(),
+            )
+            .map(|(_, full_txout)| full_txout.outpoint)
+            .collect::<BTreeSet<_>>();
+        let exp_txouts = scenario
+            .exp_chain_txouts
+            .iter()
+            .map(|(txid, vout)| OutPoint {
+                txid: *exp_tx_ids.get(txid).expect("txid must exist"),
+                vout: *vout,
+            })
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            txouts, exp_txouts,
+            "\n[{}] 'filter_chain_txouts' failed",
+            scenario.name
+        );
+
+        let utxos = tx_graph
+            .filter_chain_unspents(
+                &local_chain,
+                chain_tip,
+                spk_index.outpoints().iter().cloned(),
+            )
+            .map(|(_, full_txout)| full_txout.outpoint)
+            .collect::<BTreeSet<_>>();
+        let exp_utxos = scenario
+            .exp_unspents
+            .iter()
+            .map(|(txid, vout)| OutPoint {
+                txid: *exp_tx_ids.get(txid).expect("txid must exist"),
+                vout: *vout,
+            })
+            .collect::<BTreeSet<_>>();
+        assert_eq!(
+            utxos, exp_utxos,
+            "\n[{}] 'filter_chain_unspents' failed",
+            scenario.name
+        );
+
+        let balance = tx_graph.balance(
+            &local_chain,
+            chain_tip,
+            spk_index.outpoints().iter().cloned(),
+            |_, spk: &Script| spk_index.index_of_spk(spk).is_some(),
+        );
+        assert_eq!(
+            balance, scenario.exp_balance,
+            "\n[{}] 'balance' failed",
+            scenario.name
+        );
+    }
+}

crates/electrum/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_electrum"
-version = "0.3.0"
+version = "0.15.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,5 +12,9 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", features = ["serde", "miniscript"] }
-electrum-client = { version = "0.12" }
+bdk_chain = { path = "../chain", version = "0.16.0" }
+electrum-client = { version = "0.20" }
+#rustls = { version = "=0.21.1", optional = true, features = ["dangerous_configuration"] }
+
+[dev-dependencies]
+bdk_testenv = { path = "../testenv", default-features = false }

crates/electrum/README.md

@@ -1,3 +1,7 @@
 # BDK Electrum
 
-BDK Electrum client library for updating the keychain tracker.
+BDK Electrum extends [`electrum-client`] to update [`bdk_chain`] structures
+from an Electrum server.
+
+[`electrum-client`]: https://docs.rs/electrum-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/

crates/electrum/src/bdk_electrum_client.rs

@@ -0,0 +1,589 @@
+use bdk_chain::{
+    bitcoin::{OutPoint, ScriptBuf, Transaction, Txid},
+    collections::{BTreeMap, HashMap, HashSet},
+    local_chain::CheckPoint,
+    spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult},
+    tx_graph::TxGraph,
+    BlockId, ConfirmationHeightAnchor, ConfirmationTimeHeightAnchor,
+};
+use core::str::FromStr;
+use electrum_client::{ElectrumApi, Error, HeaderNotification};
+use std::sync::{Arc, Mutex};
+
+/// We include a chain suffix of a certain length for the purpose of robustness.
+const CHAIN_SUFFIX_LENGTH: u32 = 8;
+
+/// Wrapper around an [`electrum_client::ElectrumApi`] which includes an internal in-memory
+/// transaction cache to avoid re-fetching already downloaded transactions.
+#[derive(Debug)]
+pub struct BdkElectrumClient<E> {
+    /// The internal [`electrum_client::ElectrumApi`]
+    pub inner: E,
+    /// The transaction cache
+    tx_cache: Mutex<HashMap<Txid, Arc<Transaction>>>,
+}
+
+impl<E: ElectrumApi> BdkElectrumClient<E> {
+    /// Creates a new bdk client from a [`electrum_client::ElectrumApi`]
+    pub fn new(client: E) -> Self {
+        Self {
+            inner: client,
+            tx_cache: Default::default(),
+        }
+    }
+
+    /// Inserts transactions into the transaction cache so that the client will not fetch these
+    /// transactions.
+    pub fn populate_tx_cache<A>(&self, tx_graph: impl AsRef<TxGraph<A>>) {
+        let txs = tx_graph
+            .as_ref()
+            .full_txs()
+            .map(|tx_node| (tx_node.txid, tx_node.tx));
+
+        let mut tx_cache = self.tx_cache.lock().unwrap();
+        for (txid, tx) in txs {
+            tx_cache.insert(txid, tx);
+        }
+    }
+
+    /// Fetch transaction of given `txid`.
+    ///
+    /// If it hits the cache it will return the cached version and avoid making the request.
+    pub fn fetch_tx(&self, txid: Txid) -> Result<Arc<Transaction>, Error> {
+        let tx_cache = self.tx_cache.lock().unwrap();
+
+        if let Some(tx) = tx_cache.get(&txid) {
+            return Ok(Arc::clone(tx));
+        }
+
+        drop(tx_cache);
+
+        let tx = Arc::new(self.inner.transaction_get(&txid)?);
+
+        self.tx_cache.lock().unwrap().insert(txid, Arc::clone(&tx));
+
+        Ok(tx)
+    }
+
+    /// Broadcasts a transaction to the network.
+    ///
+    /// This is a re-export of [`ElectrumApi::transaction_broadcast`].
+    pub fn transaction_broadcast(&self, tx: &Transaction) -> Result<Txid, Error> {
+        self.inner.transaction_broadcast(tx)
+    }
+
+    /// Full scan the keychain scripts specified with the blockchain (via an Electrum client) and
+    /// returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `request`: struct with data required to perform a spk-based blockchain client full scan,
+    ///              see [`FullScanRequest`]
+    /// - `stop_gap`: the full scan for each keychain stops after a gap of script pubkeys with no
+    ///              associated transactions
+    /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch
+    ///              request
+    /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee
+    pub fn full_scan<K: Ord + Clone>(
+        &self,
+        request: FullScanRequest<K>,
+        stop_gap: usize,
+        batch_size: usize,
+        fetch_prev_txouts: bool,
+    ) -> Result<ElectrumFullScanResult<K>, Error> {
+        let mut request_spks = request.spks_by_keychain;
+
+        // We keep track of already-scanned spks just in case a reorg happens and we need to do a
+        // rescan. We need to keep track of this as iterators in `keychain_spks` are "unbounded" so
+        // cannot be collected. In addition, we keep track of whether an spk has an active tx
+        // history for determining the `last_active_index`.
+        // * key: (keychain, spk_index) that identifies the spk.
+        // * val: (script_pubkey, has_tx_history).
+        let mut scanned_spks = BTreeMap::<(K, u32), (ScriptBuf, bool)>::new();
+
+        let update = loop {
+            let (tip, _) = construct_update_tip(&self.inner, request.chain_tip.clone())?;
+            let mut graph_update = TxGraph::<ConfirmationHeightAnchor>::default();
+            let cps = tip
+                .iter()
+                .take(10)
+                .map(|cp| (cp.height(), cp))
+                .collect::<BTreeMap<u32, CheckPoint>>();
+
+            if !request_spks.is_empty() {
+                if !scanned_spks.is_empty() {
+                    scanned_spks.append(
+                        &mut self.populate_with_spks(
+                            &cps,
+                            &mut graph_update,
+                            &mut scanned_spks
+                                .iter()
+                                .map(|(i, (spk, _))| (i.clone(), spk.clone())),
+                            stop_gap,
+                            batch_size,
+                        )?,
+                    );
+                }
+                for (keychain, keychain_spks) in &mut request_spks {
+                    scanned_spks.extend(
+                        self.populate_with_spks(
+                            &cps,
+                            &mut graph_update,
+                            keychain_spks,
+                            stop_gap,
+                            batch_size,
+                        )?
+                        .into_iter()
+                        .map(|(spk_i, spk)| ((keychain.clone(), spk_i), spk)),
+                    );
+                }
+            }
+
+            // check for reorgs during scan process
+            let server_blockhash = self.inner.block_header(tip.height() as usize)?.block_hash();
+            if tip.hash() != server_blockhash {
+                continue; // reorg
+            }
+
+            // Fetch previous `TxOut`s for fee calculation if flag is enabled.
+            if fetch_prev_txouts {
+                self.fetch_prev_txout(&mut graph_update)?;
+            }
+
+            let chain_update = tip;
+
+            let keychain_update = request_spks
+                .into_keys()
+                .filter_map(|k| {
+                    scanned_spks
+                        .range((k.clone(), u32::MIN)..=(k.clone(), u32::MAX))
+                        .rev()
+                        .find(|(_, (_, active))| *active)
+                        .map(|((_, i), _)| (k, *i))
+                })
+                .collect::<BTreeMap<_, _>>();
+
+            break FullScanResult {
+                graph_update,
+                chain_update,
+                last_active_indices: keychain_update,
+            };
+        };
+
+        Ok(ElectrumFullScanResult(update))
+    }
+
+    /// Sync a set of scripts with the blockchain (via an Electrum client) for the data specified
+    /// and returns updates for [`bdk_chain`] data structures.
+    ///
+    /// - `request`: struct with data required to perform a spk-based blockchain client sync,
+    ///              see [`SyncRequest`]
+    /// - `batch_size`: specifies the max number of script pubkeys to request for in a single batch
+    ///              request
+    /// - `fetch_prev_txouts`: specifies whether or not we want previous `TxOut`s for fee
+    ///              calculation
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: Self::full_scan
+    pub fn sync(
+        &self,
+        request: SyncRequest,
+        batch_size: usize,
+        fetch_prev_txouts: bool,
+    ) -> Result<ElectrumSyncResult, Error> {
+        let full_scan_req = FullScanRequest::from_chain_tip(request.chain_tip.clone())
+            .set_spks_for_keychain((), request.spks.enumerate().map(|(i, spk)| (i as u32, spk)));
+        let mut full_scan_res = self
+            .full_scan(full_scan_req, usize::MAX, batch_size, false)?
+            .with_confirmation_height_anchor();
+
+        let (tip, _) = construct_update_tip(&self.inner, request.chain_tip)?;
+        let cps = tip
+            .iter()
+            .take(10)
+            .map(|cp| (cp.height(), cp))
+            .collect::<BTreeMap<u32, CheckPoint>>();
+
+        self.populate_with_txids(&cps, &mut full_scan_res.graph_update, request.txids)?;
+        self.populate_with_outpoints(&cps, &mut full_scan_res.graph_update, request.outpoints)?;
+
+        // Fetch previous `TxOut`s for fee calculation if flag is enabled.
+        if fetch_prev_txouts {
+            self.fetch_prev_txout(&mut full_scan_res.graph_update)?;
+        }
+
+        Ok(ElectrumSyncResult(SyncResult {
+            chain_update: full_scan_res.chain_update,
+            graph_update: full_scan_res.graph_update,
+        }))
+    }
+
+    /// Populate the `graph_update` with transactions/anchors associated with the given `spks`.
+    ///
+    /// Transactions that contains an output with requested spk, or spends form an output with
+    /// requested spk will be added to `graph_update`. Anchors of the aforementioned transactions are
+    /// also included.
+    ///
+    /// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory.
+    fn populate_with_spks<I: Ord + Clone>(
+        &self,
+        cps: &BTreeMap<u32, CheckPoint>,
+        graph_update: &mut TxGraph<ConfirmationHeightAnchor>,
+        spks: &mut impl Iterator<Item = (I, ScriptBuf)>,
+        stop_gap: usize,
+        batch_size: usize,
+    ) -> Result<BTreeMap<I, (ScriptBuf, bool)>, Error> {
+        let mut unused_spk_count = 0_usize;
+        let mut scanned_spks = BTreeMap::new();
+
+        loop {
+            let spks = (0..batch_size)
+                .map_while(|_| spks.next())
+                .collect::<Vec<_>>();
+            if spks.is_empty() {
+                return Ok(scanned_spks);
+            }
+
+            let spk_histories = self
+                .inner
+                .batch_script_get_history(spks.iter().map(|(_, s)| s.as_script()))?;
+
+            for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {
+                if spk_history.is_empty() {
+                    scanned_spks.insert(spk_index, (spk, false));
+                    unused_spk_count += 1;
+                    if unused_spk_count > stop_gap {
+                        return Ok(scanned_spks);
+                    }
+                    continue;
+                } else {
+                    scanned_spks.insert(spk_index, (spk, true));
+                    unused_spk_count = 0;
+                }
+
+                for tx_res in spk_history {
+                    let _ = graph_update.insert_tx(self.fetch_tx(tx_res.tx_hash)?);
+                    if let Some(anchor) = determine_tx_anchor(cps, tx_res.height, tx_res.tx_hash) {
+                        let _ = graph_update.insert_anchor(tx_res.tx_hash, anchor);
+                    }
+                }
+            }
+        }
+    }
+
+    // Helper function which fetches the `TxOut`s of our relevant transactions' previous transactions,
+    // which we do not have by default. This data is needed to calculate the transaction fee.
+    fn fetch_prev_txout(
+        &self,
+        graph_update: &mut TxGraph<ConfirmationHeightAnchor>,
+    ) -> Result<(), Error> {
+        let full_txs: Vec<Arc<Transaction>> =
+            graph_update.full_txs().map(|tx_node| tx_node.tx).collect();
+        for tx in full_txs {
+            for vin in &tx.input {
+                let outpoint = vin.previous_output;
+                let vout = outpoint.vout;
+                let prev_tx = self.fetch_tx(outpoint.txid)?;
+                let txout = prev_tx.output[vout as usize].clone();
+                let _ = graph_update.insert_txout(outpoint, txout);
+            }
+        }
+        Ok(())
+    }
+
+    /// Populate the `graph_update` with associated transactions/anchors of `outpoints`.
+    ///
+    /// Transactions in which the outpoint resides, and transactions that spend from the outpoint are
+    /// included. Anchors of the aforementioned transactions are included.
+    ///
+    /// Checkpoints (in `cps`) are used to create anchors. The `tx_cache` is self-explanatory.
+    fn populate_with_outpoints(
+        &self,
+        cps: &BTreeMap<u32, CheckPoint>,
+        graph_update: &mut TxGraph<ConfirmationHeightAnchor>,
+        outpoints: impl IntoIterator<Item = OutPoint>,
+    ) -> Result<(), Error> {
+        for outpoint in outpoints {
+            let op_txid = outpoint.txid;
+            let op_tx = self.fetch_tx(op_txid)?;
+            let op_txout = match op_tx.output.get(outpoint.vout as usize) {
+                Some(txout) => txout,
+                None => continue,
+            };
+            debug_assert_eq!(op_tx.compute_txid(), op_txid);
+
+            // attempt to find the following transactions (alongside their chain positions), and
+            // add to our sparsechain `update`:
+            let mut has_residing = false; // tx in which the outpoint resides
+            let mut has_spending = false; // tx that spends the outpoint
+            for res in self.inner.script_get_history(&op_txout.script_pubkey)? {
+                if has_residing && has_spending {
+                    break;
+                }
+
+                if !has_residing && res.tx_hash == op_txid {
+                    has_residing = true;
+                    let _ = graph_update.insert_tx(Arc::clone(&op_tx));
+                    if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) {
+                        let _ = graph_update.insert_anchor(res.tx_hash, anchor);
+                    }
+                }
+
+                if !has_spending && res.tx_hash != op_txid {
+                    let res_tx = self.fetch_tx(res.tx_hash)?;
+                    // we exclude txs/anchors that do not spend our specified outpoint(s)
+                    has_spending = res_tx
+                        .input
+                        .iter()
+                        .any(|txin| txin.previous_output == outpoint);
+                    if !has_spending {
+                        continue;
+                    }
+                    let _ = graph_update.insert_tx(Arc::clone(&res_tx));
+                    if let Some(anchor) = determine_tx_anchor(cps, res.height, res.tx_hash) {
+                        let _ = graph_update.insert_anchor(res.tx_hash, anchor);
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Populate the `graph_update` with transactions/anchors of the provided `txids`.
+    fn populate_with_txids(
+        &self,
+        cps: &BTreeMap<u32, CheckPoint>,
+        graph_update: &mut TxGraph<ConfirmationHeightAnchor>,
+        txids: impl IntoIterator<Item = Txid>,
+    ) -> Result<(), Error> {
+        for txid in txids {
+            let tx = match self.fetch_tx(txid) {
+                Ok(tx) => tx,
+                Err(electrum_client::Error::Protocol(_)) => continue,
+                Err(other_err) => return Err(other_err),
+            };
+
+            let spk = tx
+                .output
+                .first()
+                .map(|txo| &txo.script_pubkey)
+                .expect("tx must have an output");
+
+            // because of restrictions of the Electrum API, we have to use the `script_get_history`
+            // call to get confirmation status of our transaction
+            let anchor = match self
+                .inner
+                .script_get_history(spk)?
+                .into_iter()
+                .find(|r| r.tx_hash == txid)
+            {
+                Some(r) => determine_tx_anchor(cps, r.height, txid),
+                None => continue,
+            };
+
+            let _ = graph_update.insert_tx(tx);
+            if let Some(anchor) = anchor {
+                let _ = graph_update.insert_anchor(txid, anchor);
+            }
+        }
+        Ok(())
+    }
+}
+
+/// The result of [`BdkElectrumClient::full_scan`].
+///
+/// This can be transformed into a [`FullScanResult`] with either [`ConfirmationHeightAnchor`] or
+/// [`ConfirmationTimeHeightAnchor`] anchor types.
+pub struct ElectrumFullScanResult<K>(FullScanResult<K, ConfirmationHeightAnchor>);
+
+impl<K> ElectrumFullScanResult<K> {
+    /// Return [`FullScanResult`] with [`ConfirmationHeightAnchor`].
+    pub fn with_confirmation_height_anchor(self) -> FullScanResult<K, ConfirmationHeightAnchor> {
+        self.0
+    }
+
+    /// Return [`FullScanResult`] with [`ConfirmationTimeHeightAnchor`].
+    ///
+    /// This requires additional calls to the Electrum server.
+    pub fn with_confirmation_time_height_anchor(
+        self,
+        client: &BdkElectrumClient<impl ElectrumApi>,
+    ) -> Result<FullScanResult<K, ConfirmationTimeHeightAnchor>, Error> {
+        let res = self.0;
+        Ok(FullScanResult {
+            graph_update: try_into_confirmation_time_result(res.graph_update, &client.inner)?,
+            chain_update: res.chain_update,
+            last_active_indices: res.last_active_indices,
+        })
+    }
+}
+
+/// The result of [`BdkElectrumClient::sync`].
+///
+/// This can be transformed into a [`SyncResult`] with either [`ConfirmationHeightAnchor`] or
+/// [`ConfirmationTimeHeightAnchor`] anchor types.
+pub struct ElectrumSyncResult(SyncResult<ConfirmationHeightAnchor>);
+
+impl ElectrumSyncResult {
+    /// Return [`SyncResult`] with [`ConfirmationHeightAnchor`].
+    pub fn with_confirmation_height_anchor(self) -> SyncResult<ConfirmationHeightAnchor> {
+        self.0
+    }
+
+    /// Return [`SyncResult`] with [`ConfirmationTimeHeightAnchor`].
+    ///
+    /// This requires additional calls to the Electrum server.
+    pub fn with_confirmation_time_height_anchor(
+        self,
+        client: &BdkElectrumClient<impl ElectrumApi>,
+    ) -> Result<SyncResult<ConfirmationTimeHeightAnchor>, Error> {
+        let res = self.0;
+        Ok(SyncResult {
+            graph_update: try_into_confirmation_time_result(res.graph_update, &client.inner)?,
+            chain_update: res.chain_update,
+        })
+    }
+}
+
+fn try_into_confirmation_time_result(
+    graph_update: TxGraph<ConfirmationHeightAnchor>,
+    client: &impl ElectrumApi,
+) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+    let relevant_heights = graph_update
+        .all_anchors()
+        .iter()
+        .map(|(a, _)| a.confirmation_height)
+        .collect::<HashSet<_>>();
+
+    let height_to_time = relevant_heights
+        .clone()
+        .into_iter()
+        .zip(
+            client
+                .batch_block_header(relevant_heights)?
+                .into_iter()
+                .map(|bh| bh.time as u64),
+        )
+        .collect::<HashMap<u32, u64>>();
+
+    Ok(graph_update.map_anchors(|a| ConfirmationTimeHeightAnchor {
+        anchor_block: a.anchor_block,
+        confirmation_height: a.confirmation_height,
+        confirmation_time: height_to_time[&a.confirmation_height],
+    }))
+}
+
+/// Return a [`CheckPoint`] of the latest tip, that connects with `prev_tip`.
+fn construct_update_tip(
+    client: &impl ElectrumApi,
+    prev_tip: CheckPoint,
+) -> Result<(CheckPoint, Option<u32>), Error> {
+    let HeaderNotification { height, .. } = client.block_headers_subscribe()?;
+    let new_tip_height = height as u32;
+
+    // If electrum returns a tip height that is lower than our previous tip, then checkpoints do
+    // not need updating. We just return the previous tip and use that as the point of agreement.
+    if new_tip_height < prev_tip.height() {
+        return Ok((prev_tip.clone(), Some(prev_tip.height())));
+    }
+
+    // Atomically fetch the latest `CHAIN_SUFFIX_LENGTH` count of blocks from Electrum. We use this
+    // to construct our checkpoint update.
+    let mut new_blocks = {
+        let start_height = new_tip_height.saturating_sub(CHAIN_SUFFIX_LENGTH - 1);
+        let hashes = client
+            .block_headers(start_height as _, CHAIN_SUFFIX_LENGTH as _)?
+            .headers
+            .into_iter()
+            .map(|h| h.block_hash());
+        (start_height..).zip(hashes).collect::<BTreeMap<u32, _>>()
+    };
+
+    // Find the "point of agreement" (if any).
+    let agreement_cp = {
+        let mut agreement_cp = Option::<CheckPoint>::None;
+        for cp in prev_tip.iter() {
+            let cp_block = cp.block_id();
+            let hash = match new_blocks.get(&cp_block.height) {
+                Some(&hash) => hash,
+                None => {
+                    assert!(
+                        new_tip_height >= cp_block.height,
+                        "already checked that electrum's tip cannot be smaller"
+                    );
+                    let hash = client.block_header(cp_block.height as _)?.block_hash();
+                    new_blocks.insert(cp_block.height, hash);
+                    hash
+                }
+            };
+            if hash == cp_block.hash {
+                agreement_cp = Some(cp);
+                break;
+            }
+        }
+        agreement_cp
+    };
+
+    let agreement_height = agreement_cp.as_ref().map(CheckPoint::height);
+
+    let new_tip = new_blocks
+        .into_iter()
+        // Prune `new_blocks` to only include blocks that are actually new.
+        .filter(|(height, _)| Some(*height) > agreement_height)
+        .map(|(height, hash)| BlockId { height, hash })
+        .fold(agreement_cp, |prev_cp, block| {
+            Some(match prev_cp {
+                Some(cp) => cp.push(block).expect("must extend checkpoint"),
+                None => CheckPoint::new(block),
+            })
+        })
+        .expect("must have at least one checkpoint");
+
+    Ok((new_tip, agreement_height))
+}
+
+/// A [tx status] comprises of a concatenation of `tx_hash:height:`s. We transform a single one of
+/// these concatenations into a [`ConfirmationHeightAnchor`] if possible.
+///
+/// We use the lowest possible checkpoint as the anchor block (from `cps`). If an anchor block
+/// cannot be found, or the transaction is unconfirmed, [`None`] is returned.
+///
+/// [tx status](https://electrumx-spesmilo.readthedocs.io/en/latest/protocol-basics.html#status)
+fn determine_tx_anchor(
+    cps: &BTreeMap<u32, CheckPoint>,
+    raw_height: i32,
+    txid: Txid,
+) -> Option<ConfirmationHeightAnchor> {
+    // The electrum API has a weird quirk where an unconfirmed transaction is presented with a
+    // height of 0. To avoid invalid representation in our data structures, we manually set
+    // transactions residing in the genesis block to have height 0, then interpret a height of 0 as
+    // unconfirmed for all other transactions.
+    if txid
+        == Txid::from_str("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
+            .expect("must deserialize genesis coinbase txid")
+    {
+        let anchor_block = cps.values().next()?.block_id();
+        return Some(ConfirmationHeightAnchor {
+            anchor_block,
+            confirmation_height: 0,
+        });
+    }
+    match raw_height {
+        h if h <= 0 => {
+            debug_assert!(h == 0 || h == -1, "unexpected height ({}) from electrum", h);
+            None
+        }
+        h => {
+            let h = h as u32;
+            let anchor_block = cps.range(h..).next().map(|(_, cp)| cp.block_id())?;
+            if h > anchor_block.height {
+                None
+            } else {
+                Some(ConfirmationHeightAnchor {
+                    anchor_block,
+                    confirmation_height: h,
+                })
+            }
+        }
+    }
+}

crates/electrum/src/electrum_ext.rs

@@ -1,486 +0,0 @@
-use bdk_chain::{
-    bitcoin::{hashes::hex::FromHex, BlockHash, OutPoint, Script, Transaction, Txid},
-    keychain::LocalUpdate,
-    local_chain::LocalChain,
-    tx_graph::{self, TxGraph},
-    Anchor, BlockId, ConfirmationHeightAnchor, ConfirmationTimeAnchor,
-};
-use electrum_client::{Client, ElectrumApi, Error};
-use std::{
-    collections::{BTreeMap, BTreeSet, HashMap, HashSet},
-    fmt::Debug,
-};
-
-#[derive(Debug, Clone)]
-pub struct ElectrumUpdate<K, A> {
-    pub graph_update: HashMap<Txid, BTreeSet<A>>,
-    pub chain_update: LocalChain,
-    pub keychain_update: BTreeMap<K, u32>,
-}
-
-impl<K, A> Default for ElectrumUpdate<K, A> {
-    fn default() -> Self {
-        Self {
-            graph_update: Default::default(),
-            chain_update: Default::default(),
-            keychain_update: Default::default(),
-        }
-    }
-}
-
-impl<K, A: Anchor> ElectrumUpdate<K, A> {
-    pub fn missing_full_txs<A2>(&self, graph: &TxGraph<A2>) -> Vec<Txid> {
-        self.graph_update
-            .keys()
-            .filter(move |&&txid| graph.as_ref().get_tx(txid).is_none())
-            .cloned()
-            .collect()
-    }
-
-    pub fn finalize(
-        self,
-        client: &Client,
-        seen_at: Option<u64>,
-        missing: Vec<Txid>,
-    ) -> Result<LocalUpdate<K, A>, Error> {
-        let new_txs = client.batch_transaction_get(&missing)?;
-        let mut graph_update = TxGraph::<A>::new(new_txs);
-        for (txid, anchors) in self.graph_update {
-            if let Some(seen_at) = seen_at {
-                let _ = graph_update.insert_seen_at(txid, seen_at);
-            }
-            for anchor in anchors {
-                let _ = graph_update.insert_anchor(txid, anchor);
-            }
-        }
-        Ok(LocalUpdate {
-            keychain: self.keychain_update,
-            graph: graph_update,
-            chain: self.chain_update,
-        })
-    }
-}
-
-impl<K> ElectrumUpdate<K, ConfirmationHeightAnchor> {
-    /// Finalizes the [`ElectrumUpdate`] with `new_txs` and anchors of type
-    /// [`ConfirmationTimeAnchor`].
-    ///
-    /// **Note:** The confirmation time might not be precisely correct if there has been a reorg.
-    /// Electrum's API intends that we use the merkle proof API, we should change `bdk_electrum` to
-    /// use it.
-    pub fn finalize_as_confirmation_time(
-        self,
-        client: &Client,
-        seen_at: Option<u64>,
-        missing: Vec<Txid>,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let update = self.finalize(client, seen_at, missing)?;
-
-        let relevant_heights = {
-            let mut visited_heights = HashSet::new();
-            update
-                .graph
-                .all_anchors()
-                .iter()
-                .map(|(a, _)| a.confirmation_height_upper_bound())
-                .filter(move |&h| visited_heights.insert(h))
-                .collect::<Vec<_>>()
-        };
-
-        let height_to_time = relevant_heights
-            .clone()
-            .into_iter()
-            .zip(
-                client
-                    .batch_block_header(relevant_heights)?
-                    .into_iter()
-                    .map(|bh| bh.time as u64),
-            )
-            .collect::<HashMap<u32, u64>>();
-
-        let graph_additions = {
-            let old_additions = TxGraph::default().determine_additions(&update.graph);
-            tx_graph::Additions {
-                txs: old_additions.txs,
-                txouts: old_additions.txouts,
-                last_seen: old_additions.last_seen,
-                anchors: old_additions
-                    .anchors
-                    .into_iter()
-                    .map(|(height_anchor, txid)| {
-                        let confirmation_height = height_anchor.confirmation_height;
-                        let confirmation_time = height_to_time[&confirmation_height];
-                        let time_anchor = ConfirmationTimeAnchor {
-                            anchor_block: height_anchor.anchor_block,
-                            confirmation_height,
-                            confirmation_time,
-                        };
-                        (time_anchor, txid)
-                    })
-                    .collect(),
-            }
-        };
-
-        Ok(LocalUpdate {
-            keychain: update.keychain,
-            graph: {
-                let mut graph = TxGraph::default();
-                graph.apply_additions(graph_additions);
-                graph
-            },
-            chain: update.chain,
-        })
-    }
-}
-
-pub trait ElectrumExt<A> {
-    fn get_tip(&self) -> Result<(u32, BlockHash), Error>;
-
-    fn scan<K: Ord + Clone>(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        stop_gap: usize,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate<K, A>, Error>;
-
-    fn scan_without_keychain(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<Item = Script>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate<(), A>, Error> {
-        let spk_iter = misc_spks
-            .into_iter()
-            .enumerate()
-            .map(|(i, spk)| (i as u32, spk));
-
-        self.scan(
-            local_chain,
-            [((), spk_iter)].into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            batch_size,
-        )
-    }
-}
-
-impl ElectrumExt<ConfirmationHeightAnchor> for Client {
-    fn get_tip(&self) -> Result<(u32, BlockHash), Error> {
-        // TODO: unsubscribe when added to the client, or is there a better call to use here?
-        self.block_headers_subscribe()
-            .map(|data| (data.height as u32, data.header.block_hash()))
-    }
-
-    fn scan<K: Ord + Clone>(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        stop_gap: usize,
-        batch_size: usize,
-    ) -> Result<ElectrumUpdate<K, ConfirmationHeightAnchor>, Error> {
-        let mut request_spks = keychain_spks
-            .into_iter()
-            .map(|(k, s)| (k, s.into_iter()))
-            .collect::<BTreeMap<K, _>>();
-        let mut scanned_spks = BTreeMap::<(K, u32), (Script, bool)>::new();
-
-        let txids = txids.into_iter().collect::<Vec<_>>();
-        let outpoints = outpoints.into_iter().collect::<Vec<_>>();
-
-        let update = loop {
-            let mut update = ElectrumUpdate::<K, ConfirmationHeightAnchor> {
-                chain_update: prepare_chain_update(self, local_chain)?,
-                ..Default::default()
-            };
-            let anchor_block = update
-                .chain_update
-                .tip()
-                .expect("must have atleast one block");
-
-            if !request_spks.is_empty() {
-                if !scanned_spks.is_empty() {
-                    scanned_spks.append(&mut populate_with_spks(
-                        self,
-                        anchor_block,
-                        &mut update,
-                        &mut scanned_spks
-                            .iter()
-                            .map(|(i, (spk, _))| (i.clone(), spk.clone())),
-                        stop_gap,
-                        batch_size,
-                    )?);
-                }
-                for (keychain, keychain_spks) in &mut request_spks {
-                    scanned_spks.extend(
-                        populate_with_spks(
-                            self,
-                            anchor_block,
-                            &mut update,
-                            keychain_spks,
-                            stop_gap,
-                            batch_size,
-                        )?
-                        .into_iter()
-                        .map(|(spk_i, spk)| ((keychain.clone(), spk_i), spk)),
-                    );
-                }
-            }
-
-            populate_with_txids(self, anchor_block, &mut update, &mut txids.iter().cloned())?;
-
-            let _txs = populate_with_outpoints(
-                self,
-                anchor_block,
-                &mut update,
-                &mut outpoints.iter().cloned(),
-            )?;
-
-            // check for reorgs during scan process
-            let server_blockhash = self
-                .block_header(anchor_block.height as usize)?
-                .block_hash();
-            if anchor_block.hash != server_blockhash {
-                continue; // reorg
-            }
-
-            update.keychain_update = request_spks
-                .into_keys()
-                .filter_map(|k| {
-                    scanned_spks
-                        .range((k.clone(), u32::MIN)..=(k.clone(), u32::MAX))
-                        .rev()
-                        .find(|(_, (_, active))| *active)
-                        .map(|((_, i), _)| (k, *i))
-                })
-                .collect::<BTreeMap<_, _>>();
-            break update;
-        };
-
-        Ok(update)
-    }
-}
-
-/// Prepare an update "template" based on the checkpoints of the `local_chain`.
-fn prepare_chain_update(
-    client: &Client,
-    local_chain: &BTreeMap<u32, BlockHash>,
-) -> Result<LocalChain, Error> {
-    let mut update = LocalChain::default();
-
-    // Find the local chain block that is still there so our update can connect to the local chain.
-    for (&existing_height, &existing_hash) in local_chain.iter().rev() {
-        // TODO: a batch request may be safer, as a reorg that happens when we are obtaining
-        //       `block_header`s will result in inconsistencies
-        let current_hash = client.block_header(existing_height as usize)?.block_hash();
-        let _ = update
-            .insert_block(BlockId {
-                height: existing_height,
-                hash: current_hash,
-            })
-            .expect("This never errors because we are working with a fresh chain");
-
-        if current_hash == existing_hash {
-            break;
-        }
-    }
-
-    // Insert the new tip so new transactions will be accepted into the sparsechain.
-    let tip = {
-        let (height, hash) = crate::get_tip(client)?;
-        BlockId { height, hash }
-    };
-    if update.insert_block(tip).is_err() {
-        // There has been a re-org before we even begin scanning addresses.
-        // Just recursively call (this should never happen).
-        return prepare_chain_update(client, local_chain);
-    }
-
-    Ok(update)
-}
-
-fn determine_tx_anchor(
-    anchor_block: BlockId,
-    raw_height: i32,
-    txid: Txid,
-) -> Option<ConfirmationHeightAnchor> {
-    // The electrum API has a weird quirk where an unconfirmed transaction is presented with a
-    // height of 0. To avoid invalid representation in our data structures, we manually set
-    // transactions residing in the genesis block to have height 0, then interpret a height of 0 as
-    // unconfirmed for all other transactions.
-    if txid
-        == Txid::from_hex("4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b")
-            .expect("must deserialize genesis coinbase txid")
-    {
-        return Some(ConfirmationHeightAnchor {
-            anchor_block,
-            confirmation_height: 0,
-        });
-    }
-    match raw_height {
-        h if h <= 0 => {
-            debug_assert!(h == 0 || h == -1, "unexpected height ({}) from electrum", h);
-            None
-        }
-        h => {
-            let h = h as u32;
-            if h > anchor_block.height {
-                None
-            } else {
-                Some(ConfirmationHeightAnchor {
-                    anchor_block,
-                    confirmation_height: h,
-                })
-            }
-        }
-    }
-}
-
-fn populate_with_outpoints<K>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    outpoints: &mut impl Iterator<Item = OutPoint>,
-) -> Result<HashMap<Txid, Transaction>, Error> {
-    let mut full_txs = HashMap::new();
-    for outpoint in outpoints {
-        let txid = outpoint.txid;
-        let tx = client.transaction_get(&txid)?;
-        debug_assert_eq!(tx.txid(), txid);
-        let txout = match tx.output.get(outpoint.vout as usize) {
-            Some(txout) => txout,
-            None => continue,
-        };
-        // attempt to find the following transactions (alongside their chain positions), and
-        // add to our sparsechain `update`:
-        let mut has_residing = false; // tx in which the outpoint resides
-        let mut has_spending = false; // tx that spends the outpoint
-        for res in client.script_get_history(&txout.script_pubkey)? {
-            if has_residing && has_spending {
-                break;
-            }
-
-            if res.tx_hash == txid {
-                if has_residing {
-                    continue;
-                }
-                has_residing = true;
-                full_txs.insert(res.tx_hash, tx.clone());
-            } else {
-                if has_spending {
-                    continue;
-                }
-                let res_tx = match full_txs.get(&res.tx_hash) {
-                    Some(tx) => tx,
-                    None => {
-                        let res_tx = client.transaction_get(&res.tx_hash)?;
-                        full_txs.insert(res.tx_hash, res_tx);
-                        full_txs.get(&res.tx_hash).expect("just inserted")
-                    }
-                };
-                has_spending = res_tx
-                    .input
-                    .iter()
-                    .any(|txin| txin.previous_output == outpoint);
-                if !has_spending {
-                    continue;
-                }
-            };
-
-            let anchor = determine_tx_anchor(anchor_block, res.height, res.tx_hash);
-
-            let tx_entry = update.graph_update.entry(res.tx_hash).or_default();
-            if let Some(anchor) = anchor {
-                tx_entry.insert(anchor);
-            }
-        }
-    }
-    Ok(full_txs)
-}
-
-fn populate_with_txids<K>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    txids: &mut impl Iterator<Item = Txid>,
-) -> Result<(), Error> {
-    for txid in txids {
-        let tx = match client.transaction_get(&txid) {
-            Ok(tx) => tx,
-            Err(electrum_client::Error::Protocol(_)) => continue,
-            Err(other_err) => return Err(other_err),
-        };
-
-        let spk = tx
-            .output
-            .get(0)
-            .map(|txo| &txo.script_pubkey)
-            .expect("tx must have an output");
-
-        let anchor = match client
-            .script_get_history(spk)?
-            .into_iter()
-            .find(|r| r.tx_hash == txid)
-        {
-            Some(r) => determine_tx_anchor(anchor_block, r.height, txid),
-            None => continue,
-        };
-
-        let tx_entry = update.graph_update.entry(txid).or_default();
-        if let Some(anchor) = anchor {
-            tx_entry.insert(anchor);
-        }
-    }
-    Ok(())
-}
-
-fn populate_with_spks<K, I: Ord + Clone>(
-    client: &Client,
-    anchor_block: BlockId,
-    update: &mut ElectrumUpdate<K, ConfirmationHeightAnchor>,
-    spks: &mut impl Iterator<Item = (I, Script)>,
-    stop_gap: usize,
-    batch_size: usize,
-) -> Result<BTreeMap<I, (Script, bool)>, Error> {
-    let mut unused_spk_count = 0_usize;
-    let mut scanned_spks = BTreeMap::new();
-
-    loop {
-        let spks = (0..batch_size)
-            .map_while(|_| spks.next())
-            .collect::<Vec<_>>();
-        if spks.is_empty() {
-            return Ok(scanned_spks);
-        }
-
-        let spk_histories = client.batch_script_get_history(spks.iter().map(|(_, s)| s))?;
-
-        for ((spk_index, spk), spk_history) in spks.into_iter().zip(spk_histories) {
-            if spk_history.is_empty() {
-                scanned_spks.insert(spk_index, (spk, false));
-                unused_spk_count += 1;
-                if unused_spk_count > stop_gap {
-                    return Ok(scanned_spks);
-                }
-                continue;
-            } else {
-                scanned_spks.insert(spk_index, (spk, true));
-                unused_spk_count = 0;
-            }
-
-            for tx in spk_history {
-                let tx_entry = update.graph_update.entry(tx.tx_hash).or_default();
-                if let Some(anchor) = determine_tx_anchor(anchor_block, tx.height, tx.tx_hash) {
-                    tx_entry.insert(anchor);
-                }
-            }
-        }
-    }
-}

crates/electrum/src/lib.rs

@@ -1,35 +1,22 @@
-//! This crate is used for updating structures of the [`bdk_chain`] crate with data from electrum.
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Electrum server.
 //!
-//! The star of the show is the [`ElectrumExt::scan`] method, which scans for relevant blockchain
-//! data (via electrum) and outputs an [`ElectrumUpdate`].
+//! The two primary methods are [`BdkElectrumClient::sync`] and [`BdkElectrumClient::full_scan`]. In most cases
+//! [`BdkElectrumClient::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`BdkElectrumClient::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for
+//! [`bdk_chain`].
 //!
-//! An [`ElectrumUpdate`] only includes `txid`s and no full transactions. The caller is responsible
-//! for obtaining full transactions before applying. This can be done with
-//! these steps:
+//! Refer to [`example_electrum`] for a complete example.
 //!
-//! 1. Determine which full transactions are missing. The method [`missing_full_txs`] of
-//! [`ElectrumUpdate`] can be used.
-//!
-//! 2. Obtaining the full transactions. To do this via electrum, the method
-//! [`batch_transaction_get`] can be used.
-//!
-//! Refer to [`bdk_electrum_example`] for a complete example.
-//!
-//! [`ElectrumClient::scan`]: ElectrumClient::scan
-//! [`missing_full_txs`]: ElectrumUpdate::missing_full_txs
-//! [`batch_transaction_get`]: ElectrumApi::batch_transaction_get
-//! [`bdk_electrum_example`]: https://github.com/LLFourn/bdk_core_staging/tree/master/bdk_electrum_example
+//! [`example_electrum`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_electrum
+
+#![warn(missing_docs)]
+
+mod bdk_electrum_client;
+pub use bdk_electrum_client::*;
 
-use bdk_chain::bitcoin::BlockHash;
-use electrum_client::{Client, ElectrumApi, Error};
-mod electrum_ext;
 pub use bdk_chain;
 pub use electrum_client;
-pub use electrum_ext::*;
-
-fn get_tip(client: &Client) -> Result<(u32, BlockHash), Error> {
-    // TODO: unsubscribe when added to the client, or is there a better call to use here?
-    client
-        .block_headers_subscribe()
-        .map(|data| (data.height as u32, data.header.block_hash()))
-}

crates/electrum/tests/test_electrum.rs

@@ -0,0 +1,220 @@
+use bdk_chain::{
+    bitcoin::{hashes::Hash, Address, Amount, ScriptBuf, WScriptHash},
+    keychain::Balance,
+    local_chain::LocalChain,
+    spk_client::SyncRequest,
+    ConfirmationTimeHeightAnchor, IndexedTxGraph, SpkTxOutIndex,
+};
+use bdk_electrum::BdkElectrumClient;
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+
+fn get_balance(
+    recv_chain: &LocalChain,
+    recv_graph: &IndexedTxGraph<ConfirmationTimeHeightAnchor, SpkTxOutIndex<()>>,
+) -> anyhow::Result<Balance> {
+    let chain_tip = recv_chain.tip().block_id();
+    let outpoints = recv_graph.index.outpoints().clone();
+    let balance = recv_graph
+        .graph()
+        .balance(recv_chain, chain_tip, outpoints, |_, _| true);
+    Ok(balance)
+}
+
+/// Ensure that [`ElectrumExt`] can sync properly.
+///
+/// 1. Mine 101 blocks.
+/// 2. Send a tx.
+/// 3. Mine extra block to confirm sent tx.
+/// 4. Check [`Balance`] to ensure tx is confirmed.
+#[test]
+fn scan_detects_confirmed_tx() -> anyhow::Result<()> {
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+
+    // Setup addresses.
+    let addr_to_mine = env
+        .bitcoind
+        .client
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?;
+
+    // Setup receiver.
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.bitcoind.client.get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<ConfirmationTimeHeightAnchor, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // Mine some blocks.
+    env.mine_blocks(101, Some(addr_to_mine))?;
+
+    // Create transaction that is tracked by our receiver.
+    env.send(&addr_to_track, SEND_AMOUNT)?;
+
+    // Mine a block to confirm sent tx.
+    env.mine_blocks(1, None)?;
+
+    // Sync up to tip.
+    env.wait_until_electrum_sees_block()?;
+    let update = client
+        .sync(
+            SyncRequest::from_chain_tip(recv_chain.tip())
+                .chain_spks(core::iter::once(spk_to_track)),
+            5,
+            true,
+        )?
+        .with_confirmation_time_height_anchor(&client)?;
+
+    let _ = recv_chain
+        .apply_update(update.chain_update)
+        .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+    let _ = recv_graph.apply_update(update.graph_update);
+
+    // Check to see if tx is confirmed.
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT,
+            ..Balance::default()
+        },
+    );
+
+    for tx in recv_graph.graph().full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transaction's previous outputs.
+        let fee = recv_graph
+            .graph()
+            .calculate_fee(&tx.tx)
+            .expect("fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_unsigned()
+            .expect("valid `Amount`");
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    Ok(())
+}
+
+/// Ensure that confirmed txs that are reorged become unconfirmed.
+///
+/// 1. Mine 101 blocks.
+/// 2. Mine 8 blocks with a confirmed tx in each.
+/// 3. Perform 8 separate reorgs on each block with a confirmed tx.
+/// 4. Check [`Balance`] after each reorg to ensure unconfirmed amount is correct.
+#[test]
+fn tx_can_become_unconfirmed_after_reorg() -> anyhow::Result<()> {
+    const REORG_COUNT: usize = 8;
+    const SEND_AMOUNT: Amount = Amount::from_sat(10_000);
+
+    let env = TestEnv::new()?;
+    let electrum_client = electrum_client::Client::new(env.electrsd.electrum_url.as_str())?;
+    let client = BdkElectrumClient::new(electrum_client);
+
+    // Setup addresses.
+    let addr_to_mine = env
+        .bitcoind
+        .client
+        .get_new_address(None, None)?
+        .assume_checked();
+    let spk_to_track = ScriptBuf::new_p2wsh(&WScriptHash::all_zeros());
+    let addr_to_track = Address::from_script(&spk_to_track, bdk_chain::bitcoin::Network::Regtest)?;
+
+    // Setup receiver.
+    let (mut recv_chain, _) = LocalChain::from_genesis_hash(env.bitcoind.client.get_block_hash(0)?);
+    let mut recv_graph = IndexedTxGraph::<ConfirmationTimeHeightAnchor, _>::new({
+        let mut recv_index = SpkTxOutIndex::default();
+        recv_index.insert_spk((), spk_to_track.clone());
+        recv_index
+    });
+
+    // Mine some blocks.
+    env.mine_blocks(101, Some(addr_to_mine))?;
+
+    // Create transactions that are tracked by our receiver.
+    for _ in 0..REORG_COUNT {
+        env.send(&addr_to_track, SEND_AMOUNT)?;
+        env.mine_blocks(1, None)?;
+    }
+
+    // Sync up to tip.
+    env.wait_until_electrum_sees_block()?;
+    let update = client
+        .sync(
+            SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]),
+            5,
+            false,
+        )?
+        .with_confirmation_time_height_anchor(&client)?;
+
+    let _ = recv_chain
+        .apply_update(update.chain_update)
+        .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+    let _ = recv_graph.apply_update(update.graph_update.clone());
+
+    // Retain a snapshot of all anchors before reorg process.
+    let initial_anchors = update.graph_update.all_anchors();
+
+    // Check if initial balance is correct.
+    assert_eq!(
+        get_balance(&recv_chain, &recv_graph)?,
+        Balance {
+            confirmed: SEND_AMOUNT * REORG_COUNT as u64,
+            ..Balance::default()
+        },
+        "initial balance must be correct",
+    );
+
+    // Perform reorgs with different depths.
+    for depth in 1..=REORG_COUNT {
+        env.reorg_empty_blocks(depth)?;
+
+        env.wait_until_electrum_sees_block()?;
+        let update = client
+            .sync(
+                SyncRequest::from_chain_tip(recv_chain.tip()).chain_spks([spk_to_track.clone()]),
+                5,
+                false,
+            )?
+            .with_confirmation_time_height_anchor(&client)?;
+
+        let _ = recv_chain
+            .apply_update(update.chain_update)
+            .map_err(|err| anyhow::anyhow!("LocalChain update error: {:?}", err))?;
+
+        // Check to see if a new anchor is added during current reorg.
+        if !initial_anchors.is_superset(update.graph_update.all_anchors()) {
+            println!("New anchor added at reorg depth {}", depth);
+        }
+        let _ = recv_graph.apply_update(update.graph_update);
+
+        assert_eq!(
+            get_balance(&recv_chain, &recv_graph)?,
+            Balance {
+                confirmed: SEND_AMOUNT * (REORG_COUNT - depth) as u64,
+                trusted_pending: SEND_AMOUNT * depth as u64,
+                ..Balance::default()
+            },
+            "reorg_count: {}",
+            depth,
+        );
+    }
+
+    Ok(())
+}

crates/esplora/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "bdk_esplora"
-version = "0.3.0"
+version = "0.15.0"
 edition = "2021"
 homepage = "https://bitcoindevkit.org"
 repository = "https://github.com/bitcoindevkit/bdk"
@@ -12,18 +12,23 @@ readme = "README.md"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", default-features = false, features = ["serde", "miniscript"] }
-esplora-client = { version = "0.5", default-features = false }
+bdk_chain = { path = "../chain", version = "0.16.0", default-features = false }
+esplora-client = { version = "0.8.0", default-features = false }
 async-trait = { version = "0.1.66", optional = true }
 futures = { version = "0.3.26", optional = true }
 
-# use these dependencies if you need to enable their /no-std features
-bitcoin = { version = "0.29", optional = true, default-features = false }
-miniscript = { version = "9.0.0", optional = true, default-features = false }
+bitcoin = { version = "0.32.0", optional = true, default-features = false }
+miniscript = { version = "12.0.0", optional = true, default-features = false }
+
+[dev-dependencies]
+bdk_testenv = { path = "../testenv", default-features = false }
+tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros"] }
 
 [features]
-default = ["std", "async-https", "blocking"]
-std = ["bdk_chain/std"]
+default = ["std", "async-https", "blocking-https-rustls"]
+std = ["bdk_chain/std", "miniscript?/std"]
 async = ["async-trait", "futures", "esplora-client/async"]
 async-https = ["async", "esplora-client/async-https"]
+async-https-rustls = ["async", "esplora-client/async-https-rustls"]
 blocking = ["esplora-client/blocking"]
+blocking-https-rustls = ["esplora-client/blocking-https-rustls"]

crates/esplora/README.md

@@ -1,6 +1,6 @@
 # BDK Esplora
 
-BDK Esplora extends [`esplora_client`](crate::esplora_client) to update [`bdk_chain`] structures
+BDK Esplora extends [`esplora-client`] to update [`bdk_chain`] structures
 from an Esplora server.
 
 ## Usage
@@ -30,4 +30,7 @@ use bdk_esplora::EsploraExt;
 // use bdk_esplora::EsploraAsyncExt;
 ```
 
-For full examples, refer to [`example-crates/wallet_esplora`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora) (blocking) and [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async).
+For full examples, refer to [`example-crates/wallet_esplora_blocking`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_blocking) and [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async).
+
+[`esplora-client`]: https://docs.rs/esplora-client/
+[`bdk_chain`]: https://docs.rs/bdk-chain/

crates/esplora/src/async_ext.rs

@@ -1,269 +1,590 @@
+use std::collections::BTreeSet;
+
 use async_trait::async_trait;
+use bdk_chain::spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult};
 use bdk_chain::{
-    bitcoin::{BlockHash, OutPoint, Script, Txid},
+    bitcoin::{BlockHash, OutPoint, ScriptBuf, TxOut, Txid},
     collections::BTreeMap,
-    keychain::LocalUpdate,
-    BlockId, ConfirmationTimeAnchor,
+    local_chain::CheckPoint,
+    BlockId, ConfirmationTimeHeightAnchor, TxGraph,
 };
-use esplora_client::{Error, OutputStatus, TxStatus};
+use bdk_chain::{Anchor, Indexed};
+use esplora_client::{Amount, TxStatus};
 use futures::{stream::FuturesOrdered, TryStreamExt};
 
-use crate::map_confirmation_time_anchor;
+use crate::anchor_from_status;
 
-/// Trait to extend [`esplora_client::AsyncClient`] functionality.
+/// [`esplora_client::Error`]
+type Error = Box<esplora_client::Error>;
+
+/// Trait to extend the functionality of [`esplora_client::AsyncClient`].
 ///
-/// This is the async version of [`EsploraExt`]. Refer to
-/// [crate-level documentation] for more.
+/// Refer to [crate-level documentation] for more.
 ///
-/// [`EsploraExt`]: crate::EsploraExt
 /// [crate-level documentation]: crate
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 pub trait EsploraAsyncExt {
-    /// Scan the blockchain (via esplora) for the data specified and returns a
-    /// [`LocalUpdate<K, ConfirmationTimeAnchor>`].
+    /// Scan keychain scripts for transactions against Esplora, returning an update that can be
+    /// applied to the receiving structures.
     ///
-    /// - `local_chain`: the most recent block hashes present locally
-    /// - `keychain_spks`: keychains that we want to scan transactions for
-    /// - `txids`: transactions for which we want updated [`ConfirmationTimeAnchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to included in the update
+    /// - `request`: struct with data required to perform a spk-based blockchain client full scan,
+    ///              see [`FullScanRequest`]
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
-    /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
-    /// parallel.
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan<K: Ord + Clone + Send>(
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no
+    /// associated transactions. `parallel_requests` specifies the max number of HTTP requests to
+    /// make in parallel.
+    ///
+    /// ## Note
+    ///
+    /// `stop_gap` is defined as "the maximum number of consecutive unused addresses".
+    /// For example, with a `stop_gap` of  3, `full_scan` will keep scanning
+    /// until it encounters 3 consecutive script pubkeys with no associated transactions.
+    ///
+    /// This follows the same approach as other Bitcoin-related software,
+    /// such as [Electrum](https://electrum.readthedocs.io/en/latest/faq.html#what-is-the-gap-limit),
+    /// [BTCPay Server](https://docs.btcpayserver.org/FAQ/Wallet/#the-gap-limit-problem),
+    /// and [Sparrow](https://www.sparrowwallet.com/docs/faq.html#ive-restored-my-wallet-but-some-of-my-funds-are-missing).
+    ///
+    /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1.
+    async fn full_scan<K: Ord + Clone + Send>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<
-            K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, Script)> + Send> + Send,
-        >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        request: FullScanRequest<K>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error>;
+    ) -> Result<FullScanResult<K>, Error>;
 
-    /// Convenience method to call [`scan`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan`]: EsploraAsyncExt::scan
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan_without_keychain(
+    /// - `request`: struct with data required to perform a spk-based blockchain client sync, see
+    ///              [`SyncRequest`]
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraAsyncExt::full_scan
+    async fn sync(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = Script> + Send> + Send,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        request: SyncRequest,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<(), ConfirmationTimeAnchor>, Error> {
-        self.scan(
-            local_chain,
-            [(
-                (),
-                misc_spks
-                    .into_iter()
-                    .enumerate()
-                    .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
-        .await
-    }
+    ) -> Result<SyncResult, Error>;
 }
 
 #[cfg_attr(target_arch = "wasm32", async_trait(?Send))]
 #[cfg_attr(not(target_arch = "wasm32"), async_trait)]
 impl EsploraAsyncExt for esplora_client::AsyncClient {
-    #[allow(clippy::result_large_err)] // FIXME
-    async fn scan<K: Ord + Clone + Send>(
+    async fn full_scan<K: Ord + Clone + Send>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<
-            K,
-            impl IntoIterator<IntoIter = impl Iterator<Item = (u32, Script)> + Send> + Send,
-        >,
-        txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
-        outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+        request: FullScanRequest<K>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let parallel_requests = Ord::max(parallel_requests, 1);
+    ) -> Result<FullScanResult<K>, Error> {
+        let latest_blocks = fetch_latest_blocks(self).await?;
+        let (graph_update, last_active_indices) = full_scan_for_index_and_graph(
+            self,
+            request.spks_by_keychain,
+            stop_gap,
+            parallel_requests,
+        )
+        .await?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )
+        .await?;
+        Ok(FullScanResult {
+            chain_update,
+            graph_update,
+            last_active_indices,
+        })
+    }
 
-        let (mut update, tip_at_start) = loop {
-            let mut update = LocalUpdate::<K, ConfirmationTimeAnchor>::default();
-
-            for (&height, &original_hash) in local_chain.iter().rev() {
-                let update_block_id = BlockId {
-                    height,
-                    hash: self.get_block_hash(height).await?,
-                };
-                let _ = update
-                    .chain
-                    .insert_block(update_block_id)
-                    .expect("cannot repeat height here");
-                if update_block_id.hash == original_hash {
-                    break;
-                }
-            }
-
-            let tip_at_start = BlockId {
-                height: self.get_height().await?,
-                hash: self.get_tip_hash().await?,
-            };
-
-            if update.chain.insert_block(tip_at_start).is_ok() {
-                break (update, tip_at_start);
-            }
-        };
-
-        for (keychain, spks) in keychain_spks {
-            let mut spks = spks.into_iter();
-            let mut last_active_index = None;
-            let mut empty_scripts = 0;
-            type IndexWithTxs = (u32, Vec<esplora_client::Tx>);
-
-            loop {
-                let futures = (0..parallel_requests)
-                    .filter_map(|_| {
-                        let (index, script) = spks.next()?;
-                        let client = self.clone();
-                        Some(async move {
-                            let mut related_txs = client.scripthash_txs(&script, None).await?;
-
-                            let n_confirmed =
-                                related_txs.iter().filter(|tx| tx.status.confirmed).count();
-                            // esplora pages on 25 confirmed transactions. If there are 25 or more we
-                            // keep requesting to see if there's more.
-                            if n_confirmed >= 25 {
-                                loop {
-                                    let new_related_txs = client
-                                        .scripthash_txs(
-                                            &script,
-                                            Some(related_txs.last().unwrap().txid),
-                                        )
-                                        .await?;
-                                    let n = new_related_txs.len();
-                                    related_txs.extend(new_related_txs);
-                                    // we've reached the end
-                                    if n < 25 {
-                                        break;
-                                    }
-                                }
-                            }
-
-                            Result::<_, esplora_client::Error>::Ok((index, related_txs))
-                        })
-                    })
-                    .collect::<FuturesOrdered<_>>();
-
-                let n_futures = futures.len();
-
-                for (index, related_txs) in futures.try_collect::<Vec<IndexWithTxs>>().await? {
-                    if related_txs.is_empty() {
-                        empty_scripts += 1;
-                    } else {
-                        last_active_index = Some(index);
-                        empty_scripts = 0;
-                    }
-                    for tx in related_txs {
-                        let anchor = map_confirmation_time_anchor(&tx.status, tip_at_start);
-
-                        let _ = update.graph.insert_tx(tx.to_tx());
-                        if let Some(anchor) = anchor {
-                            let _ = update.graph.insert_anchor(tx.txid, anchor);
-                        }
-                    }
-                }
-
-                if n_futures == 0 || empty_scripts >= stop_gap {
-                    break;
-                }
-            }
-
-            if let Some(last_active_index) = last_active_index {
-                update.keychain.insert(keychain, last_active_index);
-            }
-        }
-
-        for txid in txids.into_iter() {
-            if update.graph.get_tx(txid).is_none() {
-                match self.get_tx(&txid).await? {
-                    Some(tx) => {
-                        let _ = update.graph.insert_tx(tx);
-                    }
-                    None => continue,
-                }
-            }
-            match self.get_tx_status(&txid).await? {
-                tx_status if tx_status.confirmed => {
-                    if let Some(anchor) = map_confirmation_time_anchor(&tx_status, tip_at_start) {
-                        let _ = update.graph.insert_anchor(txid, anchor);
-                    }
-                }
-                _ => continue,
-            }
-        }
-
-        for op in outpoints.into_iter() {
-            let mut op_txs = Vec::with_capacity(2);
-            if let (
-                Some(tx),
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                },
-            ) = (
-                self.get_tx(&op.txid).await?,
-                self.get_tx_status(&op.txid).await?,
-            ) {
-                op_txs.push((tx, tx_status));
-                if let Some(OutputStatus {
-                    txid: Some(txid),
-                    status: Some(spend_status),
-                    ..
-                }) = self.get_output_status(&op.txid, op.vout as _).await?
-                {
-                    if let Some(spend_tx) = self.get_tx(&txid).await? {
-                        op_txs.push((spend_tx, spend_status));
-                    }
-                }
-            }
-
-            for (tx, status) in op_txs {
-                let txid = tx.txid();
-                let anchor = map_confirmation_time_anchor(&status, tip_at_start);
-
-                let _ = update.graph.insert_tx(tx);
-                if let Some(anchor) = anchor {
-                    let _ = update.graph.insert_anchor(txid, anchor);
-                }
-            }
-        }
-
-        if tip_at_start.hash != self.get_block_hash(tip_at_start.height).await? {
-            // A reorg occurred, so let's find out where all the txids we found are now in the chain
-            let txids_found = update
-                .graph
-                .full_txs()
-                .map(|tx_node| tx_node.txid)
-                .collect::<Vec<_>>();
-            update.chain = EsploraAsyncExt::scan_without_keychain(
-                self,
-                local_chain,
-                [],
-                txids_found,
-                [],
-                parallel_requests,
-            )
-            .await?
-            .chain;
-        }
-
-        Ok(update)
+    async fn sync(
+        &self,
+        request: SyncRequest,
+        parallel_requests: usize,
+    ) -> Result<SyncResult, Error> {
+        let latest_blocks = fetch_latest_blocks(self).await?;
+        let graph_update = sync_for_index_and_graph(
+            self,
+            request.spks,
+            request.txids,
+            request.outpoints,
+            parallel_requests,
+        )
+        .await?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )
+        .await?;
+        Ok(SyncResult {
+            chain_update,
+            graph_update,
+        })
+    }
+}
+
+/// Fetch latest blocks from Esplora in an atomic call.
+///
+/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks AND
+/// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for
+/// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use
+/// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when
+/// alternating between chain-sources.
+async fn fetch_latest_blocks(
+    client: &esplora_client::AsyncClient,
+) -> Result<BTreeMap<u32, BlockHash>, Error> {
+    Ok(client
+        .get_blocks(None)
+        .await?
+        .into_iter()
+        .map(|b| (b.time.height, b.id))
+        .collect())
+}
+
+/// Used instead of [`esplora_client::BlockingClient::get_block_hash`].
+///
+/// This first checks the previously fetched `latest_blocks` before fetching from Esplora again.
+async fn fetch_block(
+    client: &esplora_client::AsyncClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    height: u32,
+) -> Result<Option<BlockHash>, Error> {
+    if let Some(&hash) = latest_blocks.get(&height) {
+        return Ok(Some(hash));
+    }
+
+    // We avoid fetching blocks higher than previously fetched `latest_blocks` as the local chain
+    // tip is used to signal for the last-synced-up-to-height.
+    let &tip_height = latest_blocks
+        .keys()
+        .last()
+        .expect("must have atleast one entry");
+    if height > tip_height {
+        return Ok(None);
+    }
+
+    Ok(Some(client.get_block_hash(height).await?))
+}
+
+/// Create the [`local_chain::Update`].
+///
+/// We want to have a corresponding checkpoint per anchor height. However, checkpoints fetched
+/// should not surpass `latest_blocks`.
+async fn chain_update<A: Anchor>(
+    client: &esplora_client::AsyncClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    local_tip: &CheckPoint,
+    anchors: &BTreeSet<(A, Txid)>,
+) -> Result<CheckPoint, Error> {
+    let mut point_of_agreement = None;
+    let mut conflicts = vec![];
+    for local_cp in local_tip.iter() {
+        let remote_hash = match fetch_block(client, latest_blocks, local_cp.height()).await? {
+            Some(hash) => hash,
+            None => continue,
+        };
+        if remote_hash == local_cp.hash() {
+            point_of_agreement = Some(local_cp.clone());
+            break;
+        } else {
+            // it is not strictly necessary to include all the conflicted heights (we do need the
+            // first one) but it seems prudent to make sure the updated chain's heights are a
+            // superset of the existing chain after update.
+            conflicts.push(BlockId {
+                height: local_cp.height(),
+                hash: remote_hash,
+            });
+        }
+    }
+
+    let mut tip = point_of_agreement.expect("remote esplora should have same genesis block");
+
+    tip = tip
+        .extend(conflicts.into_iter().rev())
+        .expect("evicted are in order");
+
+    for anchor in anchors {
+        let height = anchor.0.anchor_block().height;
+        if tip.get(height).is_none() {
+            let hash = match fetch_block(client, latest_blocks, height).await? {
+                Some(hash) => hash,
+                None => continue,
+            };
+            tip = tip.insert(BlockId { height, hash });
+        }
+    }
+
+    // insert the most recent blocks at the tip to make sure we update the tip and make the update
+    // robust.
+    for (&height, &hash) in latest_blocks.iter() {
+        tip = tip.insert(BlockId { height, hash });
+    }
+
+    Ok(tip)
+}
+
+/// This performs a full scan to get an update for the [`TxGraph`] and
+/// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex).
+async fn full_scan_for_index_and_graph<K: Ord + Clone + Send>(
+    client: &esplora_client::AsyncClient,
+    keychain_spks: BTreeMap<
+        K,
+        impl IntoIterator<IntoIter = impl Iterator<Item = Indexed<ScriptBuf>> + Send> + Send,
+    >,
+    stop_gap: usize,
+    parallel_requests: usize,
+) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
+    type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
+    let parallel_requests = Ord::max(parallel_requests, 1);
+    let mut graph = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+    let mut last_active_indexes = BTreeMap::<K, u32>::new();
+
+    for (keychain, spks) in keychain_spks {
+        let mut spks = spks.into_iter();
+        let mut last_index = Option::<u32>::None;
+        let mut last_active_index = Option::<u32>::None;
+
+        loop {
+            let handles = spks
+                .by_ref()
+                .take(parallel_requests)
+                .map(|(spk_index, spk)| {
+                    let client = client.clone();
+                    async move {
+                        let mut last_seen = None;
+                        let mut spk_txs = Vec::new();
+                        loop {
+                            let txs = client.scripthash_txs(&spk, last_seen).await?;
+                            let tx_count = txs.len();
+                            last_seen = txs.last().map(|tx| tx.txid);
+                            spk_txs.extend(txs);
+                            if tx_count < 25 {
+                                break Result::<_, Error>::Ok((spk_index, spk_txs));
+                            }
+                        }
+                    }
+                })
+                .collect::<FuturesOrdered<_>>();
+
+            if handles.is_empty() {
+                break;
+            }
+
+            for (index, txs) in handles.try_collect::<Vec<TxsOfSpkIndex>>().await? {
+                last_index = Some(index);
+                if !txs.is_empty() {
+                    last_active_index = Some(index);
+                }
+                for tx in txs {
+                    let _ = graph.insert_tx(tx.to_tx());
+                    if let Some(anchor) = anchor_from_status(&tx.status) {
+                        let _ = graph.insert_anchor(tx.txid, anchor);
+                    }
+
+                    let previous_outputs = tx.vin.iter().filter_map(|vin| {
+                        let prevout = vin.prevout.as_ref()?;
+                        Some((
+                            OutPoint {
+                                txid: vin.txid,
+                                vout: vin.vout,
+                            },
+                            TxOut {
+                                script_pubkey: prevout.scriptpubkey.clone(),
+                                value: Amount::from_sat(prevout.value),
+                            },
+                        ))
+                    });
+
+                    for (outpoint, txout) in previous_outputs {
+                        let _ = graph.insert_txout(outpoint, txout);
+                    }
+                }
+            }
+
+            let last_index = last_index.expect("Must be set since handles wasn't empty.");
+            let gap_limit_reached = if let Some(i) = last_active_index {
+                last_index >= i.saturating_add(stop_gap as u32)
+            } else {
+                last_index + 1 >= stop_gap as u32
+            };
+            if gap_limit_reached {
+                break;
+            }
+        }
+
+        if let Some(last_active_index) = last_active_index {
+            last_active_indexes.insert(keychain, last_active_index);
+        }
+    }
+
+    Ok((graph, last_active_indexes))
+}
+
+async fn sync_for_index_and_graph(
+    client: &esplora_client::AsyncClient,
+    misc_spks: impl IntoIterator<IntoIter = impl Iterator<Item = ScriptBuf> + Send> + Send,
+    txids: impl IntoIterator<IntoIter = impl Iterator<Item = Txid> + Send> + Send,
+    outpoints: impl IntoIterator<IntoIter = impl Iterator<Item = OutPoint> + Send> + Send,
+    parallel_requests: usize,
+) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+    let mut graph = full_scan_for_index_and_graph(
+        client,
+        [(
+            (),
+            misc_spks
+                .into_iter()
+                .enumerate()
+                .map(|(i, spk)| (i as u32, spk)),
+        )]
+        .into(),
+        usize::MAX,
+        parallel_requests,
+    )
+    .await
+    .map(|(g, _)| g)?;
+
+    let mut txids = txids.into_iter();
+    loop {
+        let handles = txids
+            .by_ref()
+            .take(parallel_requests)
+            .filter(|&txid| graph.get_tx(txid).is_none())
+            .map(|txid| {
+                let client = client.clone();
+                async move { client.get_tx_status(&txid).await.map(|s| (txid, s)) }
+            })
+            .collect::<FuturesOrdered<_>>();
+
+        if handles.is_empty() {
+            break;
+        }
+
+        for (txid, status) in handles.try_collect::<Vec<(Txid, TxStatus)>>().await? {
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = graph.insert_anchor(txid, anchor);
+            }
+        }
+    }
+
+    for op in outpoints.into_iter() {
+        if graph.get_tx(op.txid).is_none() {
+            if let Some(tx) = client.get_tx(&op.txid).await? {
+                let _ = graph.insert_tx(tx);
+            }
+            let status = client.get_tx_status(&op.txid).await?;
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = graph.insert_anchor(op.txid, anchor);
+            }
+        }
+
+        if let Some(op_status) = client.get_output_status(&op.txid, op.vout as _).await? {
+            if let Some(txid) = op_status.txid {
+                if graph.get_tx(txid).is_none() {
+                    if let Some(tx) = client.get_tx(&txid).await? {
+                        let _ = graph.insert_tx(tx);
+                    }
+                    let status = client.get_tx_status(&txid).await?;
+                    if let Some(anchor) = anchor_from_status(&status) {
+                        let _ = graph.insert_anchor(txid, anchor);
+                    }
+                }
+            }
+        }
+    }
+
+    Ok(graph)
+}
+
+#[cfg(test)]
+mod test {
+    use std::{collections::BTreeSet, time::Duration};
+
+    use bdk_chain::{
+        bitcoin::{hashes::Hash, Txid},
+        local_chain::LocalChain,
+        BlockId,
+    };
+    use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+    use esplora_client::Builder;
+
+    use crate::async_ext::{chain_update, fetch_latest_blocks};
+
+    macro_rules! h {
+        ($index:literal) => {{
+            bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes())
+        }};
+    }
+
+    /// Ensure that update does not remove heights (from original), and all anchor heights are included.
+    #[tokio::test]
+    pub async fn test_finalize_chain_update() -> anyhow::Result<()> {
+        struct TestCase<'a> {
+            name: &'a str,
+            /// Initial blockchain height to start the env with.
+            initial_env_height: u32,
+            /// Initial checkpoint heights to start with.
+            initial_cps: &'a [u32],
+            /// The final blockchain height of the env.
+            final_env_height: u32,
+            /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch
+            /// the blockhash from the env.
+            anchors: &'a [(u32, Txid)],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "chain_extends",
+                initial_env_height: 60,
+                initial_cps: &[59, 60],
+                final_env_height: 90,
+                anchors: &[],
+            },
+            TestCase {
+                name: "introduce_older_heights",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 50,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+            TestCase {
+                name: "introduce_older_heights_after_chain_extends",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 100,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("[{}] running test case: {}", i, t.name);
+
+            let env = TestEnv::new()?;
+            let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+            let client = Builder::new(base_url.as_str()).build_async()?;
+
+            // set env to `initial_env_height`
+            if let Some(to_mine) = t
+                .initial_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height().await? < t.initial_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft initial `local_chain`
+            let local_chain = {
+                let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?);
+                // force `chain_update_blocking` to add all checkpoints in `t.initial_cps`
+                let anchors = t
+                    .initial_cps
+                    .iter()
+                    .map(|&height| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            Txid::all_zeros(),
+                        ))
+                    })
+                    .collect::<anyhow::Result<BTreeSet<_>>>()?;
+                let update = chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client).await?,
+                    &chain.tip(),
+                    &anchors,
+                )
+                .await?;
+                chain.apply_update(update)?;
+                chain
+            };
+            println!("local chain height: {}", local_chain.tip().height());
+
+            // extend env chain
+            if let Some(to_mine) = t
+                .final_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height().await? < t.final_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft update
+            let update = {
+                let anchors = t
+                    .anchors
+                    .iter()
+                    .map(|&(height, txid)| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            txid,
+                        ))
+                    })
+                    .collect::<anyhow::Result<_>>()?;
+                chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client).await?,
+                    &local_chain.tip(),
+                    &anchors,
+                )
+                .await?
+            };
+
+            // apply update
+            let mut updated_local_chain = local_chain.clone();
+            updated_local_chain.apply_update(update)?;
+            println!(
+                "updated local chain height: {}",
+                updated_local_chain.tip().height()
+            );
+
+            assert!(
+                {
+                    let initial_heights = local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    let updated_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    updated_heights.is_superset(&initial_heights)
+                },
+                "heights from the initial chain must all be in the updated chain",
+            );
+
+            assert!(
+                {
+                    let exp_anchor_heights = t
+                        .anchors
+                        .iter()
+                        .map(|(h, _)| *h)
+                        .chain(t.initial_cps.iter().copied())
+                        .collect::<BTreeSet<_>>();
+                    let anchor_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    anchor_heights.is_superset(&exp_anchor_heights)
+                },
+                "anchor heights must all be in updated chain",
+            );
+        }
+
+        Ok(())
     }
 }

crates/esplora/src/blocking_ext.rs

@@ -1,251 +1,786 @@
-use bdk_chain::bitcoin::{BlockHash, OutPoint, Script, Txid};
+use std::collections::BTreeSet;
+use std::thread::JoinHandle;
+use std::usize;
+
 use bdk_chain::collections::BTreeMap;
-use bdk_chain::BlockId;
-use bdk_chain::{keychain::LocalUpdate, ConfirmationTimeAnchor};
-use esplora_client::{Error, OutputStatus, TxStatus};
+use bdk_chain::spk_client::{FullScanRequest, FullScanResult, SyncRequest, SyncResult};
+use bdk_chain::{
+    bitcoin::{Amount, BlockHash, OutPoint, ScriptBuf, TxOut, Txid},
+    local_chain::CheckPoint,
+    BlockId, ConfirmationTimeHeightAnchor, TxGraph,
+};
+use bdk_chain::{Anchor, Indexed};
+use esplora_client::TxStatus;
 
-use crate::map_confirmation_time_anchor;
+use crate::anchor_from_status;
 
-/// Trait to extend [`esplora_client::BlockingClient`] functionality.
+/// [`esplora_client::Error`]
+pub type Error = Box<esplora_client::Error>;
+
+/// Trait to extend the functionality of [`esplora_client::BlockingClient`].
 ///
 /// Refer to [crate-level documentation] for more.
 ///
 /// [crate-level documentation]: crate
 pub trait EsploraExt {
-    /// Scan the blockchain (via esplora) for the data specified and returns a
-    /// [`LocalUpdate<K, ConfirmationTimeAnchor>`].
+    /// Scan keychain scripts for transactions against Esplora, returning an update that can be
+    /// applied to the receiving structures.
     ///
-    /// - `local_chain`: the most recent block hashes present locally
-    /// - `keychain_spks`: keychains that we want to scan transactions for
-    /// - `txids`: transactions for which we want updated [`ConfirmationTimeAnchor`]s
-    /// - `outpoints`: transactions associated with these outpoints (residing, spending) that we
-    ///     want to included in the update
+    /// - `request`: struct with data required to perform a spk-based blockchain client full scan,
+    ///              see [`FullScanRequest`]
     ///
-    /// The scan for each keychain stops after a gap of `stop_gap` script pubkeys with no associated
-    /// transactions. `parallel_requests` specifies the max number of HTTP requests to make in
-    /// parallel.
-    #[allow(clippy::result_large_err)] // FIXME
-    fn scan<K: Ord + Clone>(
+    /// The full scan for each keychain stops after a gap of `stop_gap` script pubkeys with no
+    /// associated transactions. `parallel_requests` specifies the max number of HTTP requests to
+    /// make in parallel.
+    ///
+    /// ## Note
+    ///
+    /// `stop_gap` is defined as "the maximum number of consecutive unused addresses".
+    /// For example, with a `stop_gap` of  3, `full_scan` will keep scanning
+    /// until it encounters 3 consecutive script pubkeys with no associated transactions.
+    ///
+    /// This follows the same approach as other Bitcoin-related software,
+    /// such as [Electrum](https://electrum.readthedocs.io/en/latest/faq.html#what-is-the-gap-limit),
+    /// [BTCPay Server](https://docs.btcpayserver.org/FAQ/Wallet/#the-gap-limit-problem),
+    /// and [Sparrow](https://www.sparrowwallet.com/docs/faq.html#ive-restored-my-wallet-but-some-of-my-funds-are-missing).
+    ///
+    /// A `stop_gap` of 0 will be treated as a `stop_gap` of 1.
+    fn full_scan<K: Ord + Clone>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
+        request: FullScanRequest<K>,
         stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error>;
+    ) -> Result<FullScanResult<K>, Error>;
 
-    /// Convenience method to call [`scan`] without requiring a keychain.
+    /// Sync a set of scripts with the blockchain (via an Esplora client) for the data
+    /// specified and return a [`TxGraph`].
     ///
-    /// [`scan`]: EsploraExt::scan
-    #[allow(clippy::result_large_err)] // FIXME
-    fn scan_without_keychain(
+    /// - `request`: struct with data required to perform a spk-based blockchain client sync, see
+    ///              [`SyncRequest`]
+    ///
+    /// If the scripts to sync are unknown, such as when restoring or importing a keychain that
+    /// may include scripts that have been used, use [`full_scan`] with the keychain.
+    ///
+    /// [`full_scan`]: EsploraExt::full_scan
+    fn sync(&self, request: SyncRequest, parallel_requests: usize) -> Result<SyncResult, Error>;
+}
+
+impl EsploraExt for esplora_client::BlockingClient {
+    fn full_scan<K: Ord + Clone>(
         &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        misc_spks: impl IntoIterator<Item = Script>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
+        request: FullScanRequest<K>,
+        stop_gap: usize,
         parallel_requests: usize,
-    ) -> Result<LocalUpdate<(), ConfirmationTimeAnchor>, Error> {
-        self.scan(
-            local_chain,
-            [(
+    ) -> Result<FullScanResult<K>, Error> {
+        let latest_blocks = fetch_latest_blocks(self)?;
+        let (graph_update, last_active_indices) = full_scan_for_index_and_graph_blocking(
+            self,
+            request.spks_by_keychain,
+            stop_gap,
+            parallel_requests,
+        )?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )?;
+        Ok(FullScanResult {
+            chain_update,
+            graph_update,
+            last_active_indices,
+        })
+    }
+
+    fn sync(&self, request: SyncRequest, parallel_requests: usize) -> Result<SyncResult, Error> {
+        let latest_blocks = fetch_latest_blocks(self)?;
+        let graph_update = sync_for_index_and_graph_blocking(
+            self,
+            request.spks,
+            request.txids,
+            request.outpoints,
+            parallel_requests,
+        )?;
+        let chain_update = chain_update(
+            self,
+            &latest_blocks,
+            &request.chain_tip,
+            graph_update.all_anchors(),
+        )?;
+        Ok(SyncResult {
+            chain_update,
+            graph_update,
+        })
+    }
+}
+
+/// Fetch latest blocks from Esplora in an atomic call.
+///
+/// We want to do this before fetching transactions and anchors as we cannot fetch latest blocks AND
+/// transactions atomically, and the checkpoint tip is used to determine last-scanned block (for
+/// block-based chain-sources). Therefore it's better to be conservative when setting the tip (use
+/// an earlier tip rather than a later tip) otherwise the caller may accidentally skip blocks when
+/// alternating between chain-sources.
+fn fetch_latest_blocks(
+    client: &esplora_client::BlockingClient,
+) -> Result<BTreeMap<u32, BlockHash>, Error> {
+    Ok(client
+        .get_blocks(None)?
+        .into_iter()
+        .map(|b| (b.time.height, b.id))
+        .collect())
+}
+
+/// Used instead of [`esplora_client::BlockingClient::get_block_hash`].
+///
+/// This first checks the previously fetched `latest_blocks` before fetching from Esplora again.
+fn fetch_block(
+    client: &esplora_client::BlockingClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    height: u32,
+) -> Result<Option<BlockHash>, Error> {
+    if let Some(&hash) = latest_blocks.get(&height) {
+        return Ok(Some(hash));
+    }
+
+    // We avoid fetching blocks higher than previously fetched `latest_blocks` as the local chain
+    // tip is used to signal for the last-synced-up-to-height.
+    let &tip_height = latest_blocks
+        .keys()
+        .last()
+        .expect("must have atleast one entry");
+    if height > tip_height {
+        return Ok(None);
+    }
+
+    Ok(Some(client.get_block_hash(height)?))
+}
+
+/// Create the [`local_chain::Update`].
+///
+/// We want to have a corresponding checkpoint per anchor height. However, checkpoints fetched
+/// should not surpass `latest_blocks`.
+fn chain_update<A: Anchor>(
+    client: &esplora_client::BlockingClient,
+    latest_blocks: &BTreeMap<u32, BlockHash>,
+    local_tip: &CheckPoint,
+    anchors: &BTreeSet<(A, Txid)>,
+) -> Result<CheckPoint, Error> {
+    let mut point_of_agreement = None;
+    let mut conflicts = vec![];
+    for local_cp in local_tip.iter() {
+        let remote_hash = match fetch_block(client, latest_blocks, local_cp.height())? {
+            Some(hash) => hash,
+            None => continue,
+        };
+        if remote_hash == local_cp.hash() {
+            point_of_agreement = Some(local_cp.clone());
+            break;
+        } else {
+            // it is not strictly necessary to include all the conflicted heights (we do need the
+            // first one) but it seems prudent to make sure the updated chain's heights are a
+            // superset of the existing chain after update.
+            conflicts.push(BlockId {
+                height: local_cp.height(),
+                hash: remote_hash,
+            });
+        }
+    }
+
+    let mut tip = point_of_agreement.expect("remote esplora should have same genesis block");
+
+    tip = tip
+        .extend(conflicts.into_iter().rev())
+        .expect("evicted are in order");
+
+    for anchor in anchors {
+        let height = anchor.0.anchor_block().height;
+        if tip.get(height).is_none() {
+            let hash = match fetch_block(client, latest_blocks, height)? {
+                Some(hash) => hash,
+                None => continue,
+            };
+            tip = tip.insert(BlockId { height, hash });
+        }
+    }
+
+    // insert the most recent blocks at the tip to make sure we update the tip and make the update
+    // robust.
+    for (&height, &hash) in latest_blocks.iter() {
+        tip = tip.insert(BlockId { height, hash });
+    }
+
+    Ok(tip)
+}
+
+/// This performs a full scan to get an update for the [`TxGraph`] and
+/// [`KeychainTxOutIndex`](bdk_chain::keychain::KeychainTxOutIndex).
+fn full_scan_for_index_and_graph_blocking<K: Ord + Clone>(
+    client: &esplora_client::BlockingClient,
+    keychain_spks: BTreeMap<K, impl IntoIterator<Item = Indexed<ScriptBuf>>>,
+    stop_gap: usize,
+    parallel_requests: usize,
+) -> Result<(TxGraph<ConfirmationTimeHeightAnchor>, BTreeMap<K, u32>), Error> {
+    type TxsOfSpkIndex = (u32, Vec<esplora_client::Tx>);
+    let parallel_requests = Ord::max(parallel_requests, 1);
+    let mut tx_graph = TxGraph::<ConfirmationTimeHeightAnchor>::default();
+    let mut last_active_indices = BTreeMap::<K, u32>::new();
+
+    for (keychain, spks) in keychain_spks {
+        let mut spks = spks.into_iter();
+        let mut last_index = Option::<u32>::None;
+        let mut last_active_index = Option::<u32>::None;
+
+        loop {
+            let handles = spks
+                .by_ref()
+                .take(parallel_requests)
+                .map(|(spk_index, spk)| {
+                    std::thread::spawn({
+                        let client = client.clone();
+                        move || -> Result<TxsOfSpkIndex, Error> {
+                            let mut last_seen = None;
+                            let mut spk_txs = Vec::new();
+                            loop {
+                                let txs = client.scripthash_txs(&spk, last_seen)?;
+                                let tx_count = txs.len();
+                                last_seen = txs.last().map(|tx| tx.txid);
+                                spk_txs.extend(txs);
+                                if tx_count < 25 {
+                                    break Ok((spk_index, spk_txs));
+                                }
+                            }
+                        }
+                    })
+                })
+                .collect::<Vec<JoinHandle<Result<TxsOfSpkIndex, Error>>>>();
+
+            if handles.is_empty() {
+                break;
+            }
+
+            for handle in handles {
+                let (index, txs) = handle.join().expect("thread must not panic")?;
+                last_index = Some(index);
+                if !txs.is_empty() {
+                    last_active_index = Some(index);
+                }
+                for tx in txs {
+                    let _ = tx_graph.insert_tx(tx.to_tx());
+                    if let Some(anchor) = anchor_from_status(&tx.status) {
+                        let _ = tx_graph.insert_anchor(tx.txid, anchor);
+                    }
+
+                    let previous_outputs = tx.vin.iter().filter_map(|vin| {
+                        let prevout = vin.prevout.as_ref()?;
+                        Some((
+                            OutPoint {
+                                txid: vin.txid,
+                                vout: vin.vout,
+                            },
+                            TxOut {
+                                script_pubkey: prevout.scriptpubkey.clone(),
+                                value: Amount::from_sat(prevout.value),
+                            },
+                        ))
+                    });
+
+                    for (outpoint, txout) in previous_outputs {
+                        let _ = tx_graph.insert_txout(outpoint, txout);
+                    }
+                }
+            }
+
+            let last_index = last_index.expect("Must be set since handles wasn't empty.");
+            let gap_limit_reached = if let Some(i) = last_active_index {
+                last_index >= i.saturating_add(stop_gap as u32)
+            } else {
+                last_index + 1 >= stop_gap as u32
+            };
+            if gap_limit_reached {
+                break;
+            }
+        }
+
+        if let Some(last_active_index) = last_active_index {
+            last_active_indices.insert(keychain, last_active_index);
+        }
+    }
+
+    Ok((tx_graph, last_active_indices))
+}
+
+fn sync_for_index_and_graph_blocking(
+    client: &esplora_client::BlockingClient,
+    misc_spks: impl IntoIterator<Item = ScriptBuf>,
+    txids: impl IntoIterator<Item = Txid>,
+    outpoints: impl IntoIterator<Item = OutPoint>,
+    parallel_requests: usize,
+) -> Result<TxGraph<ConfirmationTimeHeightAnchor>, Error> {
+    let (mut tx_graph, _) = full_scan_for_index_and_graph_blocking(
+        client,
+        {
+            let mut keychains = BTreeMap::new();
+            keychains.insert(
                 (),
                 misc_spks
                     .into_iter()
                     .enumerate()
                     .map(|(i, spk)| (i as u32, spk)),
-            )]
-            .into(),
-            txids,
-            outpoints,
-            usize::MAX,
-            parallel_requests,
-        )
+            );
+            keychains
+        },
+        usize::MAX,
+        parallel_requests,
+    )?;
+
+    let mut txids = txids.into_iter();
+    loop {
+        let handles = txids
+            .by_ref()
+            .take(parallel_requests)
+            .filter(|&txid| tx_graph.get_tx(txid).is_none())
+            .map(|txid| {
+                std::thread::spawn({
+                    let client = client.clone();
+                    move || {
+                        client
+                            .get_tx_status(&txid)
+                            .map_err(Box::new)
+                            .map(|s| (txid, s))
+                    }
+                })
+            })
+            .collect::<Vec<JoinHandle<Result<(Txid, TxStatus), Error>>>>();
+
+        if handles.is_empty() {
+            break;
+        }
+
+        for handle in handles {
+            let (txid, status) = handle.join().expect("thread must not panic")?;
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = tx_graph.insert_anchor(txid, anchor);
+            }
+        }
     }
-}
 
-impl EsploraExt for esplora_client::BlockingClient {
-    fn scan<K: Ord + Clone>(
-        &self,
-        local_chain: &BTreeMap<u32, BlockHash>,
-        keychain_spks: BTreeMap<K, impl IntoIterator<Item = (u32, Script)>>,
-        txids: impl IntoIterator<Item = Txid>,
-        outpoints: impl IntoIterator<Item = OutPoint>,
-        stop_gap: usize,
-        parallel_requests: usize,
-    ) -> Result<LocalUpdate<K, ConfirmationTimeAnchor>, Error> {
-        let parallel_requests = Ord::max(parallel_requests, 1);
+    for op in outpoints {
+        if tx_graph.get_tx(op.txid).is_none() {
+            if let Some(tx) = client.get_tx(&op.txid)? {
+                let _ = tx_graph.insert_tx(tx);
+            }
+            let status = client.get_tx_status(&op.txid)?;
+            if let Some(anchor) = anchor_from_status(&status) {
+                let _ = tx_graph.insert_anchor(op.txid, anchor);
+            }
+        }
 
-        let (mut update, tip_at_start) = loop {
-            let mut update = LocalUpdate::<K, ConfirmationTimeAnchor>::default();
-
-            for (&height, &original_hash) in local_chain.iter().rev() {
-                let update_block_id = BlockId {
-                    height,
-                    hash: self.get_block_hash(height)?,
-                };
-                let _ = update
-                    .chain
-                    .insert_block(update_block_id)
-                    .expect("cannot repeat height here");
-                if update_block_id.hash == original_hash {
-                    break;
+        if let Some(op_status) = client.get_output_status(&op.txid, op.vout as _)? {
+            if let Some(txid) = op_status.txid {
+                if tx_graph.get_tx(txid).is_none() {
+                    if let Some(tx) = client.get_tx(&txid)? {
+                        let _ = tx_graph.insert_tx(tx);
+                    }
+                    let status = client.get_tx_status(&txid)?;
+                    if let Some(anchor) = anchor_from_status(&status) {
+                        let _ = tx_graph.insert_anchor(txid, anchor);
+                    }
                 }
             }
+        }
+    }
 
-            let tip_at_start = BlockId {
-                height: self.get_height()?,
-                hash: self.get_tip_hash()?,
+    Ok(tx_graph)
+}
+
+#[cfg(test)]
+mod test {
+    use crate::blocking_ext::{chain_update, fetch_latest_blocks};
+    use bdk_chain::bitcoin::hashes::Hash;
+    use bdk_chain::bitcoin::Txid;
+    use bdk_chain::local_chain::LocalChain;
+    use bdk_chain::BlockId;
+    use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+    use esplora_client::{BlockHash, Builder};
+    use std::collections::{BTreeMap, BTreeSet};
+    use std::time::Duration;
+
+    macro_rules! h {
+        ($index:literal) => {{
+            bdk_chain::bitcoin::hashes::Hash::hash($index.as_bytes())
+        }};
+    }
+
+    macro_rules! local_chain {
+        [ $(($height:expr, $block_hash:expr)), * ] => {{
+            #[allow(unused_mut)]
+            bdk_chain::local_chain::LocalChain::from_blocks([$(($height, $block_hash).into()),*].into_iter().collect())
+                .expect("chain must have genesis block")
+        }};
+    }
+
+    /// Ensure that update does not remove heights (from original), and all anchor heights are included.
+    #[test]
+    pub fn test_finalize_chain_update() -> anyhow::Result<()> {
+        struct TestCase<'a> {
+            name: &'a str,
+            /// Initial blockchain height to start the env with.
+            initial_env_height: u32,
+            /// Initial checkpoint heights to start with in the local chain.
+            initial_cps: &'a [u32],
+            /// The final blockchain height of the env.
+            final_env_height: u32,
+            /// The anchors to test with: `(height, txid)`. Only the height is provided as we can fetch
+            /// the blockhash from the env.
+            anchors: &'a [(u32, Txid)],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "chain_extends",
+                initial_env_height: 60,
+                initial_cps: &[59, 60],
+                final_env_height: 90,
+                anchors: &[],
+            },
+            TestCase {
+                name: "introduce_older_heights",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 50,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+            TestCase {
+                name: "introduce_older_heights_after_chain_extends",
+                initial_env_height: 50,
+                initial_cps: &[10, 15],
+                final_env_height: 100,
+                anchors: &[(11, h!("A")), (14, h!("B"))],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("[{}] running test case: {}", i, t.name);
+
+            let env = TestEnv::new()?;
+            let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+            let client = Builder::new(base_url.as_str()).build_blocking();
+
+            // set env to `initial_env_height`
+            if let Some(to_mine) = t
+                .initial_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height()? < t.initial_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft initial `local_chain`
+            let local_chain = {
+                let (mut chain, _) = LocalChain::from_genesis_hash(env.genesis_hash()?);
+                // force `chain_update_blocking` to add all checkpoints in `t.initial_cps`
+                let anchors = t
+                    .initial_cps
+                    .iter()
+                    .map(|&height| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            Txid::all_zeros(),
+                        ))
+                    })
+                    .collect::<anyhow::Result<BTreeSet<_>>>()?;
+                let update = chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client)?,
+                    &chain.tip(),
+                    &anchors,
+                )?;
+                chain.apply_update(update)?;
+                chain
+            };
+            println!("local chain height: {}", local_chain.tip().height());
+
+            // extend env chain
+            if let Some(to_mine) = t
+                .final_env_height
+                .checked_sub(env.make_checkpoint_tip().height())
+            {
+                env.mine_blocks(to_mine as _, None)?;
+            }
+            while client.get_height()? < t.final_env_height {
+                std::thread::sleep(Duration::from_millis(10));
+            }
+
+            // craft update
+            let update = {
+                let anchors = t
+                    .anchors
+                    .iter()
+                    .map(|&(height, txid)| -> anyhow::Result<_> {
+                        Ok((
+                            BlockId {
+                                height,
+                                hash: env.bitcoind.client.get_block_hash(height as _)?,
+                            },
+                            txid,
+                        ))
+                    })
+                    .collect::<anyhow::Result<_>>()?;
+                chain_update(
+                    &client,
+                    &fetch_latest_blocks(&client)?,
+                    &local_chain.tip(),
+                    &anchors,
+                )?
             };
 
-            if update.chain.insert_block(tip_at_start).is_ok() {
-                break (update, tip_at_start);
-            }
-        };
+            // apply update
+            let mut updated_local_chain = local_chain.clone();
+            updated_local_chain.apply_update(update)?;
+            println!(
+                "updated local chain height: {}",
+                updated_local_chain.tip().height()
+            );
 
-        for (keychain, spks) in keychain_spks {
-            let mut spks = spks.into_iter();
-            let mut last_active_index = None;
-            let mut empty_scripts = 0;
-            type IndexWithTxs = (u32, Vec<esplora_client::Tx>);
-
-            loop {
-                let handles = (0..parallel_requests)
-                    .filter_map(
-                        |_| -> Option<std::thread::JoinHandle<Result<IndexWithTxs, _>>> {
-                            let (index, script) = spks.next()?;
-                            let client = self.clone();
-                            Some(std::thread::spawn(move || {
-                                let mut related_txs = client.scripthash_txs(&script, None)?;
-
-                                let n_confirmed =
-                                    related_txs.iter().filter(|tx| tx.status.confirmed).count();
-                                // esplora pages on 25 confirmed transactions. If there are 25 or more we
-                                // keep requesting to see if there's more.
-                                if n_confirmed >= 25 {
-                                    loop {
-                                        let new_related_txs = client.scripthash_txs(
-                                            &script,
-                                            Some(related_txs.last().unwrap().txid),
-                                        )?;
-                                        let n = new_related_txs.len();
-                                        related_txs.extend(new_related_txs);
-                                        // we've reached the end
-                                        if n < 25 {
-                                            break;
-                                        }
-                                    }
-                                }
-
-                                Result::<_, esplora_client::Error>::Ok((index, related_txs))
-                            }))
-                        },
-                    )
-                    .collect::<Vec<_>>();
-
-                let n_handles = handles.len();
-
-                for handle in handles {
-                    let (index, related_txs) = handle.join().unwrap()?; // TODO: don't unwrap
-                    if related_txs.is_empty() {
-                        empty_scripts += 1;
-                    } else {
-                        last_active_index = Some(index);
-                        empty_scripts = 0;
-                    }
-                    for tx in related_txs {
-                        let anchor = map_confirmation_time_anchor(&tx.status, tip_at_start);
-
-                        let _ = update.graph.insert_tx(tx.to_tx());
-                        if let Some(anchor) = anchor {
-                            let _ = update.graph.insert_anchor(tx.txid, anchor);
-                        }
-                    }
-                }
-
-                if n_handles == 0 || empty_scripts >= stop_gap {
-                    break;
-                }
-            }
-
-            if let Some(last_active_index) = last_active_index {
-                update.keychain.insert(keychain, last_active_index);
-            }
-        }
-
-        for txid in txids.into_iter() {
-            if update.graph.get_tx(txid).is_none() {
-                match self.get_tx(&txid)? {
-                    Some(tx) => {
-                        let _ = update.graph.insert_tx(tx);
-                    }
-                    None => continue,
-                }
-            }
-            match self.get_tx_status(&txid)? {
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                } => {
-                    if let Some(anchor) = map_confirmation_time_anchor(&tx_status, tip_at_start) {
-                        let _ = update.graph.insert_anchor(txid, anchor);
-                    }
-                }
-                _ => continue,
-            }
-        }
-
-        for op in outpoints.into_iter() {
-            let mut op_txs = Vec::with_capacity(2);
-            if let (
-                Some(tx),
-                tx_status @ TxStatus {
-                    confirmed: true, ..
-                },
-            ) = (self.get_tx(&op.txid)?, self.get_tx_status(&op.txid)?)
-            {
-                op_txs.push((tx, tx_status));
-                if let Some(OutputStatus {
-                    txid: Some(txid),
-                    status: Some(spend_status),
-                    ..
-                }) = self.get_output_status(&op.txid, op.vout as _)?
+            assert!(
                 {
-                    if let Some(spend_tx) = self.get_tx(&txid)? {
-                        op_txs.push((spend_tx, spend_status));
-                    }
-                }
-            }
+                    let initial_heights = local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    let updated_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    updated_heights.is_superset(&initial_heights)
+                },
+                "heights from the initial chain must all be in the updated chain",
+            );
 
-            for (tx, status) in op_txs {
-                let txid = tx.txid();
-                let anchor = map_confirmation_time_anchor(&status, tip_at_start);
+            assert!(
+                {
+                    let exp_anchor_heights = t
+                        .anchors
+                        .iter()
+                        .map(|(h, _)| *h)
+                        .chain(t.initial_cps.iter().copied())
+                        .collect::<BTreeSet<_>>();
+                    let anchor_heights = updated_local_chain
+                        .iter_checkpoints()
+                        .map(|cp| cp.height())
+                        .collect::<BTreeSet<_>>();
+                    anchor_heights.is_superset(&exp_anchor_heights)
+                },
+                "anchor heights must all be in updated chain",
+            );
+        }
 
-                let _ = update.graph.insert_tx(tx);
-                if let Some(anchor) = anchor {
-                    let _ = update.graph.insert_anchor(txid, anchor);
-                }
+        Ok(())
+    }
+
+    #[test]
+    fn update_local_chain() -> anyhow::Result<()> {
+        const TIP_HEIGHT: u32 = 50;
+
+        let env = TestEnv::new()?;
+        let blocks = {
+            let bitcoind_client = &env.bitcoind.client;
+            assert_eq!(bitcoind_client.get_block_count()?, 1);
+            [
+                (0, bitcoind_client.get_block_hash(0)?),
+                (1, bitcoind_client.get_block_hash(1)?),
+            ]
+            .into_iter()
+            .chain((2..).zip(env.mine_blocks((TIP_HEIGHT - 1) as usize, None)?))
+            .collect::<BTreeMap<_, _>>()
+        };
+        // so new blocks can be seen by Electrs
+        let env = env.reset_electrsd()?;
+        let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+        let client = Builder::new(base_url.as_str()).build_blocking();
+
+        struct TestCase {
+            name: &'static str,
+            /// Original local chain to start off with.
+            chain: LocalChain,
+            /// Heights of floating anchors. [`chain_update_blocking`] will request for checkpoints
+            /// of these heights.
+            request_heights: &'static [u32],
+            /// The expected local chain result (heights only).
+            exp_update_heights: &'static [u32],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "request_later_blocks",
+                chain: local_chain![(0, blocks[&0]), (21, blocks[&21])],
+                request_heights: &[22, 25, 28],
+                exp_update_heights: &[21, 22, 25, 28],
+            },
+            TestCase {
+                name: "request_prev_blocks",
+                chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (5, blocks[&5])],
+                request_heights: &[4],
+                exp_update_heights: &[4, 5],
+            },
+            TestCase {
+                name: "request_prev_blocks_2",
+                chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (10, blocks[&10])],
+                request_heights: &[4, 6],
+                exp_update_heights: &[4, 6, 10],
+            },
+            TestCase {
+                name: "request_later_and_prev_blocks",
+                chain: local_chain![(0, blocks[&0]), (7, blocks[&7]), (11, blocks[&11])],
+                request_heights: &[8, 9, 15],
+                exp_update_heights: &[8, 9, 11, 15],
+            },
+            TestCase {
+                name: "request_tip_only",
+                chain: local_chain![(0, blocks[&0]), (5, blocks[&5]), (49, blocks[&49])],
+                request_heights: &[TIP_HEIGHT],
+                exp_update_heights: &[49],
+            },
+            TestCase {
+                name: "request_nothing",
+                chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, blocks[&23])],
+                request_heights: &[],
+                exp_update_heights: &[23],
+            },
+            TestCase {
+                name: "request_nothing_during_reorg",
+                chain: local_chain![(0, blocks[&0]), (13, blocks[&13]), (23, h!("23"))],
+                request_heights: &[],
+                exp_update_heights: &[13, 23],
+            },
+            TestCase {
+                name: "request_nothing_during_reorg_2",
+                chain: local_chain![
+                    (0, blocks[&0]),
+                    (21, blocks[&21]),
+                    (22, h!("22")),
+                    (23, h!("23"))
+                ],
+                request_heights: &[],
+                exp_update_heights: &[21, 22, 23],
+            },
+            TestCase {
+                name: "request_prev_blocks_during_reorg",
+                chain: local_chain![
+                    (0, blocks[&0]),
+                    (21, blocks[&21]),
+                    (22, h!("22")),
+                    (23, h!("23"))
+                ],
+                request_heights: &[17, 20],
+                exp_update_heights: &[17, 20, 21, 22, 23],
+            },
+            TestCase {
+                name: "request_later_blocks_during_reorg",
+                chain: local_chain![
+                    (0, blocks[&0]),
+                    (9, blocks[&9]),
+                    (22, h!("22")),
+                    (23, h!("23"))
+                ],
+                request_heights: &[25, 27],
+                exp_update_heights: &[9, 22, 23, 25, 27],
+            },
+            TestCase {
+                name: "request_later_blocks_during_reorg_2",
+                chain: local_chain![(0, blocks[&0]), (9, h!("9"))],
+                request_heights: &[10],
+                exp_update_heights: &[0, 9, 10],
+            },
+            TestCase {
+                name: "request_later_and_prev_blocks_during_reorg",
+                chain: local_chain![(0, blocks[&0]), (1, blocks[&1]), (9, h!("9"))],
+                request_heights: &[8, 11],
+                exp_update_heights: &[1, 8, 9, 11],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("Case {}: {}", i, t.name);
+            let mut chain = t.chain;
+
+            let mock_anchors = t
+                .request_heights
+                .iter()
+                .map(|&h| {
+                    let anchor_blockhash: BlockHash = bdk_chain::bitcoin::hashes::Hash::hash(
+                        &format!("hash_at_height_{}", h).into_bytes(),
+                    );
+                    let txid: Txid = bdk_chain::bitcoin::hashes::Hash::hash(
+                        &format!("txid_at_height_{}", h).into_bytes(),
+                    );
+                    let anchor = BlockId {
+                        height: h,
+                        hash: anchor_blockhash,
+                    };
+                    (anchor, txid)
+                })
+                .collect::<BTreeSet<_>>();
+            let chain_update = chain_update(
+                &client,
+                &fetch_latest_blocks(&client)?,
+                &chain.tip(),
+                &mock_anchors,
+            )?;
+
+            let update_blocks = chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+
+            let exp_update_blocks = t
+                .exp_update_heights
+                .iter()
+                .map(|&height| {
+                    let hash = blocks[&height];
+                    BlockId { height, hash }
+                })
+                .chain(
+                    // Electrs Esplora `get_block` call fetches 10 blocks which is included in the
+                    // update
+                    blocks
+                        .range(TIP_HEIGHT - 9..)
+                        .map(|(&height, &hash)| BlockId { height, hash }),
+                )
+                .collect::<BTreeSet<_>>();
+
+            assert!(
+                update_blocks.is_superset(&exp_update_blocks),
+                "[{}:{}] unexpected update",
+                i,
+                t.name
+            );
+
+            let _ = chain
+                .apply_update(chain_update)
+                .unwrap_or_else(|err| panic!("[{}:{}] update failed to apply: {}", i, t.name, err));
+
+            // all requested heights must exist in the final chain
+            for height in t.request_heights {
+                let exp_blockhash = blocks.get(height).expect("block must exist in bitcoind");
+                assert_eq!(
+                    chain.get(*height).map(|cp| cp.hash()),
+                    Some(*exp_blockhash),
+                    "[{}:{}] block {}:{} must exist in final chain",
+                    i,
+                    t.name,
+                    height,
+                    exp_blockhash
+                );
             }
         }
 
-        if tip_at_start.hash != self.get_block_hash(tip_at_start.height)? {
-            // A reorg occurred, so let's find out where all the txids we found are now in the chain
-            let txids_found = update
-                .graph
-                .full_txs()
-                .map(|tx_node| tx_node.txid)
-                .collect::<Vec<_>>();
-            update.chain = EsploraExt::scan_without_keychain(
-                self,
-                local_chain,
-                [],
-                txids_found,
-                [],
-                parallel_requests,
-            )?
-            .chain;
-        }
-
-        Ok(update)
+        Ok(())
     }
 }

crates/esplora/src/lib.rs

@@ -1,5 +1,22 @@
 #![doc = include_str!("../README.md")]
-use bdk_chain::{BlockId, ConfirmationTimeAnchor};
+
+//! This crate is used for updating structures of [`bdk_chain`] with data from an Esplora server.
+//!
+//! The two primary methods are [`EsploraExt::sync`] and [`EsploraExt::full_scan`]. In most cases
+//! [`EsploraExt::sync`] is used to sync the transaction histories of scripts that the application
+//! cares about, for example the scripts for all the receive addresses of a Wallet's keychain that it
+//! has shown a user. [`EsploraExt::full_scan`] is meant to be used when importing or restoring a
+//! keychain where the range of possibly used scripts is not known. In this case it is necessary to
+//! scan all keychain scripts until a number (the "stop gap") of unused scripts is discovered. For a
+//! sync or full scan the user receives relevant blockchain data and output updates for [`bdk_chain`]
+//! via a new [`TxGraph`] to be appended to any existing [`TxGraph`] data.
+//!
+//! Refer to [`example_esplora`] for a complete example.
+//!
+//! [`TxGraph`]: bdk_chain::tx_graph::TxGraph
+//! [`example_esplora`]: https://github.com/bitcoindevkit/bdk/tree/master/example-crates/example_esplora
+
+use bdk_chain::{BlockId, ConfirmationTimeHeightAnchor};
 use esplora_client::TxStatus;
 
 pub use esplora_client;
@@ -14,16 +31,20 @@ mod async_ext;
 #[cfg(feature = "async")]
 pub use async_ext::*;
 
-pub(crate) fn map_confirmation_time_anchor(
-    tx_status: &TxStatus,
-    tip_at_start: BlockId,
-) -> Option<ConfirmationTimeAnchor> {
-    match (tx_status.block_time, tx_status.block_height) {
-        (Some(confirmation_time), Some(confirmation_height)) => Some(ConfirmationTimeAnchor {
-            anchor_block: tip_at_start,
-            confirmation_height,
-            confirmation_time,
-        }),
-        _ => None,
+fn anchor_from_status(status: &TxStatus) -> Option<ConfirmationTimeHeightAnchor> {
+    if let TxStatus {
+        block_height: Some(height),
+        block_hash: Some(hash),
+        block_time: Some(time),
+        ..
+    } = status.clone()
+    {
+        Some(ConfirmationTimeHeightAnchor {
+            anchor_block: BlockId { height, hash },
+            confirmation_height: height,
+            confirmation_time: time,
+        })
+    } else {
+        None
     }
 }

crates/esplora/tests/async_ext.rs

@@ -0,0 +1,231 @@
+use bdk_chain::spk_client::{FullScanRequest, SyncRequest};
+use bdk_esplora::EsploraAsyncExt;
+use esplora_client::{self, Builder};
+use std::collections::{BTreeSet, HashSet};
+use std::str::FromStr;
+use std::thread::sleep;
+use std::time::Duration;
+
+use bdk_chain::bitcoin::{Address, Amount, Txid};
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+
+#[tokio::test]
+pub async fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_async()?;
+
+    let receive_address0 =
+        Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
+    let receive_address1 =
+        Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked();
+
+    let misc_spks = [
+        receive_address0.script_pubkey(),
+        receive_address1.script_pubkey(),
+    ];
+
+    let _block_hashes = env.mine_blocks(101, None)?;
+    let txid1 = env.bitcoind.client.send_to_address(
+        &receive_address1,
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let txid2 = env.bitcoind.client.send_to_address(
+        &receive_address0,
+        Amount::from_sat(20000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().await.unwrap() < 102 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    let sync_update = {
+        let request = SyncRequest::from_chain_tip(cp_tip.clone()).set_spks(misc_spks);
+        client.sync(request, 1).await?
+    };
+
+    assert!(
+        {
+            let update_cps = sync_update
+                .chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            let superset_cps = cp_tip
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            superset_cps.is_superset(&update_cps)
+        },
+        "update should not alter original checkpoint tip since we already started with all checkpoints",
+    );
+
+    let graph_update = sync_update.graph_update;
+    // Check to see if we have the floating txouts available from our two created transactions'
+    // previous outputs in order to calculate transaction fees.
+    for tx in graph_update.full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transactions' previous outputs.
+        let fee = graph_update.calculate_fee(&tx.tx).expect("Fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_unsigned()
+            .expect("valid `Amount`");
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    let mut graph_update_txids: Vec<Txid> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    graph_update_txids.sort();
+    let mut expected_txids = vec![txid1, txid2];
+    expected_txids.sort();
+    assert_eq!(graph_update_txids, expected_txids);
+    Ok(())
+}
+
+/// Test the bounds of the address scan depending on the `stop_gap`.
+#[tokio::test]
+pub async fn test_async_update_tx_graph_stop_gap() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_async()?;
+    let _block_hashes = env.mine_blocks(101, None)?;
+
+    // Now let's test the gap limit. First of all get a chain of 10 addresses.
+    let addresses = [
+        "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4",
+        "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30",
+        "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g",
+        "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww",
+        "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu",
+        "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh",
+        "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2",
+        "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8",
+        "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef",
+        "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk",
+    ];
+    let addresses: Vec<_> = addresses
+        .into_iter()
+        .map(|s| Address::from_str(s).unwrap().assume_checked())
+        .collect();
+    let spks: Vec<_> = addresses
+        .iter()
+        .enumerate()
+        .map(|(i, addr)| (i as u32, addr.script_pubkey()))
+        .collect();
+
+    // Then receive coins on the 4th address.
+    let txid_4th_addr = env.bitcoind.client.send_to_address(
+        &addresses[3],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().await.unwrap() < 103 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    // A scan with a gap limit of 3 won't find the transaction, but a scan with a gap limit of 4
+    // will.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 3, 1).await?
+    };
+    assert!(full_scan_update.graph_update.full_txs().next().is_none());
+    assert!(full_scan_update.last_active_indices.is_empty());
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 4, 1).await?
+    };
+    assert_eq!(
+        full_scan_update
+            .graph_update
+            .full_txs()
+            .next()
+            .unwrap()
+            .txid,
+        txid_4th_addr
+    );
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+
+    // Now receive a coin on the last address.
+    let txid_last_addr = env.bitcoind.client.send_to_address(
+        &addresses[addresses.len() - 1],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().await.unwrap() < 104 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will.
+    // The last active indice won't be updated in the first case but will in the second one.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 5, 1).await?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 1);
+    assert!(txs.contains(&txid_4th_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 6, 1).await?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 2);
+    assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 9);
+
+    Ok(())
+}

crates/esplora/tests/blocking_ext.rs

@@ -0,0 +1,232 @@
+use bdk_chain::spk_client::{FullScanRequest, SyncRequest};
+use bdk_esplora::EsploraExt;
+use esplora_client::{self, Builder};
+use std::collections::{BTreeSet, HashSet};
+use std::str::FromStr;
+use std::thread::sleep;
+use std::time::Duration;
+
+use bdk_chain::bitcoin::{Address, Amount, Txid};
+use bdk_testenv::{anyhow, bitcoincore_rpc::RpcApi, TestEnv};
+
+#[test]
+pub fn test_update_tx_graph_without_keychain() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_blocking();
+
+    let receive_address0 =
+        Address::from_str("bcrt1qc6fweuf4xjvz4x3gx3t9e0fh4hvqyu2qw4wvxm")?.assume_checked();
+    let receive_address1 =
+        Address::from_str("bcrt1qfjg5lv3dvc9az8patec8fjddrs4aqtauadnagr")?.assume_checked();
+
+    let misc_spks = [
+        receive_address0.script_pubkey(),
+        receive_address1.script_pubkey(),
+    ];
+
+    let _block_hashes = env.mine_blocks(101, None)?;
+    let txid1 = env.bitcoind.client.send_to_address(
+        &receive_address1,
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let txid2 = env.bitcoind.client.send_to_address(
+        &receive_address0,
+        Amount::from_sat(20000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().unwrap() < 102 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    let sync_update = {
+        let request = SyncRequest::from_chain_tip(cp_tip.clone()).set_spks(misc_spks);
+        client.sync(request, 1)?
+    };
+
+    assert!(
+        {
+            let update_cps = sync_update
+                .chain_update
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            let superset_cps = cp_tip
+                .iter()
+                .map(|cp| cp.block_id())
+                .collect::<BTreeSet<_>>();
+            superset_cps.is_superset(&update_cps)
+        },
+        "update should not alter original checkpoint tip since we already started with all checkpoints",
+    );
+
+    let graph_update = sync_update.graph_update;
+    // Check to see if we have the floating txouts available from our two created transactions'
+    // previous outputs in order to calculate transaction fees.
+    for tx in graph_update.full_txs() {
+        // Retrieve the calculated fee from `TxGraph`, which will panic if we do not have the
+        // floating txouts available from the transactions' previous outputs.
+        let fee = graph_update.calculate_fee(&tx.tx).expect("Fee must exist");
+
+        // Retrieve the fee in the transaction data from `bitcoind`.
+        let tx_fee = env
+            .bitcoind
+            .client
+            .get_transaction(&tx.txid, None)
+            .expect("Tx must exist")
+            .fee
+            .expect("Fee must exist")
+            .abs()
+            .to_unsigned()
+            .expect("valid `Amount`");
+
+        // Check that the calculated fee matches the fee from the transaction data.
+        assert_eq!(fee, tx_fee);
+    }
+
+    let mut graph_update_txids: Vec<Txid> = graph_update.full_txs().map(|tx| tx.txid).collect();
+    graph_update_txids.sort();
+    let mut expected_txids = vec![txid1, txid2];
+    expected_txids.sort();
+    assert_eq!(graph_update_txids, expected_txids);
+
+    Ok(())
+}
+
+/// Test the bounds of the address scan depending on the `stop_gap`.
+#[test]
+pub fn test_update_tx_graph_stop_gap() -> anyhow::Result<()> {
+    let env = TestEnv::new()?;
+    let base_url = format!("http://{}", &env.electrsd.esplora_url.clone().unwrap());
+    let client = Builder::new(base_url.as_str()).build_blocking();
+    let _block_hashes = env.mine_blocks(101, None)?;
+
+    // Now let's test the gap limit. First of all get a chain of 10 addresses.
+    let addresses = [
+        "bcrt1qj9f7r8r3p2y0sqf4r3r62qysmkuh0fzep473d2ar7rcz64wqvhssjgf0z4",
+        "bcrt1qmm5t0ch7vh2hryx9ctq3mswexcugqe4atkpkl2tetm8merqkthas3w7q30",
+        "bcrt1qut9p7ej7l7lhyvekj28xknn8gnugtym4d5qvnp5shrsr4nksmfqsmyn87g",
+        "bcrt1qqz0xtn3m235p2k96f5wa2dqukg6shxn9n3txe8arlrhjh5p744hsd957ww",
+        "bcrt1q9c0t62a8l6wfytmf2t9lfj35avadk3mm8g4p3l84tp6rl66m48sqrme7wu",
+        "bcrt1qkmh8yrk2v47cklt8dytk8f3ammcwa4q7dzattedzfhqzvfwwgyzsg59zrh",
+        "bcrt1qvgrsrzy07gjkkfr5luplt0azxtfwmwq5t62gum5jr7zwcvep2acs8hhnp2",
+        "bcrt1qw57edarcg50ansq8mk3guyrk78rk0fwvrds5xvqeupteu848zayq549av8",
+        "bcrt1qvtve5ekf6e5kzs68knvnt2phfw6a0yjqrlgat392m6zt9jsvyxhqfx67ef",
+        "bcrt1qw03ddumfs9z0kcu76ln7jrjfdwam20qtffmkcral3qtza90sp9kqm787uk",
+    ];
+    let addresses: Vec<_> = addresses
+        .into_iter()
+        .map(|s| Address::from_str(s).unwrap().assume_checked())
+        .collect();
+    let spks: Vec<_> = addresses
+        .iter()
+        .enumerate()
+        .map(|(i, addr)| (i as u32, addr.script_pubkey()))
+        .collect();
+
+    // Then receive coins on the 4th address.
+    let txid_4th_addr = env.bitcoind.client.send_to_address(
+        &addresses[3],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().unwrap() < 103 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // use a full checkpoint linked list (since this is not what we are testing)
+    let cp_tip = env.make_checkpoint_tip();
+
+    // A scan with a stop_gap of 3 won't find the transaction, but a scan with a gap limit of 4
+    // will.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 3, 1)?
+    };
+    assert!(full_scan_update.graph_update.full_txs().next().is_none());
+    assert!(full_scan_update.last_active_indices.is_empty());
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 4, 1)?
+    };
+    assert_eq!(
+        full_scan_update
+            .graph_update
+            .full_txs()
+            .next()
+            .unwrap()
+            .txid,
+        txid_4th_addr
+    );
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+
+    // Now receive a coin on the last address.
+    let txid_last_addr = env.bitcoind.client.send_to_address(
+        &addresses[addresses.len() - 1],
+        Amount::from_sat(10000),
+        None,
+        None,
+        None,
+        None,
+        Some(1),
+        None,
+    )?;
+    let _block_hashes = env.mine_blocks(1, None)?;
+    while client.get_height().unwrap() < 104 {
+        sleep(Duration::from_millis(10))
+    }
+
+    // A scan with gap limit 5 won't find the second transaction, but a scan with gap limit 6 will.
+    // The last active indice won't be updated in the first case but will in the second one.
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 5, 1)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 1);
+    assert!(txs.contains(&txid_4th_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 3);
+    let full_scan_update = {
+        let request =
+            FullScanRequest::from_chain_tip(cp_tip.clone()).set_spks_for_keychain(0, spks.clone());
+        client.full_scan(request, 6, 1)?
+    };
+    let txs: HashSet<_> = full_scan_update
+        .graph_update
+        .full_txs()
+        .map(|tx| tx.txid)
+        .collect();
+    assert_eq!(txs.len(), 2);
+    assert!(txs.contains(&txid_4th_addr) && txs.contains(&txid_last_addr));
+    assert_eq!(full_scan_update.last_active_indices[&0], 9);
+
+    Ok(())
+}

crates/file_store/Cargo.toml

@@ -1,17 +1,17 @@
 [package]
 name = "bdk_file_store"
-version = "0.2.0"
+version = "0.13.0"
 edition = "2021"
 license = "MIT OR Apache-2.0"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk_file_store"
-description = "A simple append-only flat file implementation of Persist for Bitcoin Dev Kit."
+description = "A simple append-only flat file database for persisting bdk_chain data."
 keywords = ["bitcoin", "persist", "persistence", "bdk", "file"]
 authors = ["Bitcoin Dev Kit Developers"]
 readme = "README.md"
 
 [dependencies]
-bdk_chain = { path = "../chain", version = "0.5.0", features = [ "serde", "miniscript" ] }
+bdk_chain = { path = "../chain", version = "0.16.0", features = [ "serde", "miniscript" ] }
 bincode = { version = "1" }
 serde = { version = "1", features = ["derive"] }
 

crates/file_store/README.md

@@ -1,10 +1,7 @@
 # BDK File Store
 
-This is a simple append-only flat file implementation of
-[`Persist`](`bdk_chain::Persist`).
+This is a simple append-only flat file database for persisting [`bdk_chain`] changesets.
 
-The main structure is [`Store`](`crate::Store`), which can be used with [`bdk`]'s
-`Wallet` to persist wallet data into a flat file.
+The main structure is [`Store`] which works with any [`bdk_chain`] based changesets to persist data into a flat file.
 
-[`bdk`]: https://docs.rs/bdk/latest
-[`bdk_chain`]: https://docs.rs/bdk_chain/latest
+[`bdk_chain`]:https://docs.rs/bdk_chain/latest/bdk_chain/

crates/file_store/src/entry_iter.rs

@@ -1,7 +1,7 @@
 use bincode::Options;
 use std::{
     fs::File,
-    io::{self, Seek},
+    io::{self, BufReader, Seek},
     marker::PhantomData,
 };
 
@@ -14,8 +14,9 @@ use crate::bincode_options;
 ///
 /// [`next`]: Self::next
 pub struct EntryIter<'t, T> {
-    db_file: Option<&'t mut File>,
-
+    /// Buffered reader around the file
+    db_file: BufReader<&'t mut File>,
+    finished: bool,
     /// The file position for the first read of `db_file`.
     start_pos: Option<u64>,
     types: PhantomData<T>,
@@ -24,8 +25,9 @@ pub struct EntryIter<'t, T> {
 impl<'t, T> EntryIter<'t, T> {
     pub fn new(start_pos: u64, db_file: &'t mut File) -> Self {
         Self {
-            db_file: Some(db_file),
+            db_file: BufReader::new(db_file),
             start_pos: Some(start_pos),
+            finished: false,
             types: PhantomData,
         }
     }
@@ -38,44 +40,44 @@ where
     type Item = Result<T, IterError>;
 
     fn next(&mut self) -> Option<Self::Item> {
-        // closure which reads a single entry starting from `self.pos`
-        let read_one = |f: &mut File, start_pos: Option<u64>| -> Result<Option<T>, IterError> {
-            let pos = match start_pos {
-                Some(pos) => f.seek(io::SeekFrom::Start(pos))?,
-                None => f.stream_position()?,
-            };
+        if self.finished {
+            return None;
+        }
+        (|| {
+            if let Some(start) = self.start_pos.take() {
+                self.db_file.seek(io::SeekFrom::Start(start))?;
+            }
 
-            match bincode_options().deserialize_from(&*f) {
-                Ok(changeset) => {
-                    f.stream_position()?;
-                    Ok(Some(changeset))
-                }
+            let pos_before_read = self.db_file.stream_position()?;
+            match bincode_options().deserialize_from(&mut self.db_file) {
+                Ok(changeset) => Ok(Some(changeset)),
                 Err(e) => {
+                    self.finished = true;
+                    let pos_after_read = self.db_file.stream_position()?;
+                    // allow unexpected EOF if 0 bytes were read
                     if let bincode::ErrorKind::Io(inner) = &*e {
-                        if inner.kind() == io::ErrorKind::UnexpectedEof {
-                            let eof = f.seek(io::SeekFrom::End(0))?;
-                            if pos == eof {
-                                return Ok(None);
-                            }
+                        if inner.kind() == io::ErrorKind::UnexpectedEof
+                            && pos_after_read == pos_before_read
+                        {
+                            return Ok(None);
                         }
                     }
-                    f.seek(io::SeekFrom::Start(pos))?;
+                    self.db_file.seek(io::SeekFrom::Start(pos_before_read))?;
                     Err(IterError::Bincode(*e))
                 }
             }
-        };
-
-        let result = read_one(self.db_file.as_mut()?, self.start_pos.take());
-        if result.is_err() {
-            self.db_file = None;
-        }
-        result.transpose()
+        })()
+        .transpose()
     }
 }
 
-impl From<io::Error> for IterError {
-    fn from(value: io::Error) -> Self {
-        IterError::Io(value)
+impl<'t, T> Drop for EntryIter<'t, T> {
+    fn drop(&mut self) {
+        // This syncs the underlying file's offset with the buffer's position. This way, we
+        // maintain the correct position to start the next read/write.
+        if let Ok(pos) = self.db_file.stream_position() {
+            let _ = self.db_file.get_mut().seek(io::SeekFrom::Start(pos));
+        }
     }
 }
 
@@ -97,4 +99,10 @@ impl core::fmt::Display for IterError {
     }
 }
 
+impl From<io::Error> for IterError {
+    fn from(value: io::Error) -> Self {
+        IterError::Io(value)
+    }
+}
+
 impl std::error::Error for IterError {}

crates/file_store/src/lib.rs

@@ -13,14 +13,14 @@ pub(crate) fn bincode_options() -> impl bincode::Options {
 
 /// Error that occurs due to problems encountered with the file.
 #[derive(Debug)]
-pub enum FileError<'a> {
+pub enum FileError {
     /// IO error, this may mean that the file is too short.
     Io(io::Error),
     /// Magic bytes do not match what is expected.
-    InvalidMagicBytes { got: Vec<u8>, expected: &'a [u8] },
+    InvalidMagicBytes { got: Vec<u8>, expected: Vec<u8> },
 }
 
-impl<'a> core::fmt::Display for FileError<'a> {
+impl core::fmt::Display for FileError {
     fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
         match self {
             Self::Io(e) => write!(f, "io error trying to read file: {}", e),
@@ -33,10 +33,10 @@ impl<'a> core::fmt::Display for FileError<'a> {
     }
 }
 
-impl<'a> From<io::Error> for FileError<'a> {
+impl From<io::Error> for FileError {
     fn from(value: io::Error) -> Self {
         Self::Io(value)
     }
 }
 
-impl<'a> std::error::Error for FileError<'a> {}
+impl std::error::Error for FileError {}

crates/file_store/src/store.rs

@@ -1,100 +1,112 @@
+use crate::{bincode_options, EntryIter, FileError, IterError};
+use bdk_chain::Append;
+use bincode::Options;
 use std::{
-    fmt::Debug,
+    fmt::{self, Debug},
     fs::{File, OpenOptions},
     io::{self, Read, Seek, Write},
     marker::PhantomData,
     path::Path,
 };
 
-use bdk_chain::{Append, PersistBackend};
-use bincode::Options;
-
-use crate::{bincode_options, EntryIter, FileError, IterError};
-
 /// Persists an append-only list of changesets (`C`) to a single file.
-///
-/// The changesets are the results of altering a tracker implementation (`T`).
 #[derive(Debug)]
-pub struct Store<'a, C> {
-    magic: &'a [u8],
+pub struct Store<C>
+where
+    C: Sync + Send,
+{
+    magic_len: usize,
     db_file: File,
     marker: PhantomData<C>,
 }
 
-impl<'a, C> PersistBackend<C> for Store<'a, C>
+impl<C> Store<C>
 where
-    C: Default + Append + serde::Serialize + serde::de::DeserializeOwned,
+    C: Append
+        + serde::Serialize
+        + serde::de::DeserializeOwned
+        + core::marker::Send
+        + core::marker::Sync,
 {
-    type WriteError = std::io::Error;
-
-    type LoadError = IterError;
-
-    fn write_changes(&mut self, changeset: &C) -> Result<(), Self::WriteError> {
-        self.append_changeset(changeset)
-    }
-
-    fn load_from_persistence(&mut self) -> Result<C, Self::LoadError> {
-        let (changeset, result) = self.aggregate_changesets();
-        result.map(|_| changeset)
-    }
-}
-
-impl<'a, C> Store<'a, C>
-where
-    C: Default + Append + serde::Serialize + serde::de::DeserializeOwned,
-{
-    /// Creates a new store from a [`File`].
+    /// Create a new [`Store`] file in write-only mode; error if the file exists.
     ///
-    /// The file must have been opened with read and write permissions.
+    /// `magic` is the prefixed bytes to write to the new file. This will be checked when opening
+    /// the `Store` in the future with [`open`].
     ///
-    /// `magic` is the expected prefixed bytes of the file. If this does not match, an error will be
-    /// returned.
-    ///
-    /// [`File`]: std::fs::File
-    pub fn new(magic: &'a [u8], mut db_file: File) -> Result<Self, FileError> {
-        db_file.rewind()?;
-
-        let mut magic_buf = vec![0_u8; magic.len()];
-        db_file.read_exact(magic_buf.as_mut())?;
-
-        if magic_buf != magic {
-            return Err(FileError::InvalidMagicBytes {
-                got: magic_buf,
-                expected: magic,
-            });
+    /// [`open`]: Store::open
+    pub fn create_new<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
+    where
+        P: AsRef<Path>,
+    {
+        if file_path.as_ref().exists() {
+            // `io::Error` is used instead of a variant on `FileError` because there is already a
+            // nightly-only `File::create_new` method
+            return Err(FileError::Io(io::Error::new(
+                io::ErrorKind::Other,
+                "file already exists",
+            )));
         }
-
+        let mut f = OpenOptions::new()
+            .create(true)
+            .read(true)
+            .write(true)
+            .truncate(true)
+            .open(file_path)?;
+        f.write_all(magic)?;
         Ok(Self {
-            magic,
-            db_file,
+            magic_len: magic.len(),
+            db_file: f,
             marker: Default::default(),
         })
     }
 
-    /// Creates or loads a store from `db_path`.
+    /// Open an existing [`Store`].
     ///
-    /// If no file exists there, it will be created.
+    /// Use [`create_new`] to create a new `Store`.
     ///
-    /// Refer to [`new`] for documentation on the `magic` input.
+    /// # Errors
     ///
-    /// [`new`]: Self::new
-    pub fn new_from_path<P>(magic: &'a [u8], db_path: P) -> Result<Self, FileError>
+    /// If the prefixed bytes of the opened file does not match the provided `magic`, the
+    /// [`FileError::InvalidMagicBytes`] error variant will be returned.
+    ///
+    /// [`create_new`]: Store::create_new
+    pub fn open<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
     where
         P: AsRef<Path>,
     {
-        let already_exists = db_path.as_ref().exists();
+        let mut f = OpenOptions::new().read(true).write(true).open(file_path)?;
 
-        let mut db_file = OpenOptions::new()
-            .read(true)
-            .write(true)
-            .create(true)
-            .open(db_path)?;
-
-        if !already_exists {
-            db_file.write_all(magic)?;
+        let mut magic_buf = vec![0_u8; magic.len()];
+        f.read_exact(&mut magic_buf)?;
+        if magic_buf != magic {
+            return Err(FileError::InvalidMagicBytes {
+                got: magic_buf,
+                expected: magic.to_vec(),
+            });
         }
 
-        Self::new(magic, db_file)
+        Ok(Self {
+            magic_len: magic.len(),
+            db_file: f,
+            marker: Default::default(),
+        })
+    }
+
+    /// Attempt to open existing [`Store`] file; create it if the file is non-existent.
+    ///
+    /// Internally, this calls either [`open`] or [`create_new`].
+    ///
+    /// [`open`]: Store::open
+    /// [`create_new`]: Store::create_new
+    pub fn open_or_create_new<P>(magic: &[u8], file_path: P) -> Result<Self, FileError>
+    where
+        P: AsRef<Path>,
+    {
+        if file_path.as_ref().exists() {
+            Self::open(magic, file_path)
+        } else {
+            Self::create_new(magic, file_path)
+        }
     }
 
     /// Iterates over the stored changeset from first to last, changing the seek position at each
@@ -107,31 +119,39 @@ where
     /// always iterate over all entries until `None` is returned if you want your next write to go
     /// at the end; otherwise, you will write over existing entries.
     pub fn iter_changesets(&mut self) -> EntryIter<C> {
-        EntryIter::new(self.magic.len() as u64, &mut self.db_file)
+        EntryIter::new(self.magic_len as u64, &mut self.db_file)
     }
 
     /// Loads all the changesets that have been stored as one giant changeset.
     ///
-    /// This function returns a tuple of the aggregate changeset and a result that indicates
-    /// whether an error occurred while reading or deserializing one of the entries. If so the
-    /// changeset will consist of all of those it was able to read.
+    /// This function returns the aggregate changeset, or `None` if nothing was persisted.
+    /// If reading or deserializing any of the entries fails, an error is returned that
+    /// consists of all those it was able to read.
     ///
     /// You should usually check the error. In many applications, it may make sense to do a full
     /// wallet scan with a stop-gap after getting an error, since it is likely that one of the
-    /// changesets it was unable to read changed the derivation indices of the tracker.
+    /// changesets was unable to read changes of the derivation indices of a keychain.
     ///
     /// **WARNING**: This method changes the write position of the underlying file. The next
     /// changeset will be written over the erroring entry (or the end of the file if none existed).
-    pub fn aggregate_changesets(&mut self) -> (C, Result<(), IterError>) {
-        let mut changeset = C::default();
-        let result = (|| {
-            for next_changeset in self.iter_changesets() {
-                changeset.append(next_changeset?);
+    pub fn aggregate_changesets(&mut self) -> Result<Option<C>, AggregateChangesetsError<C>> {
+        let mut changeset = Option::<C>::None;
+        for next_changeset in self.iter_changesets() {
+            let next_changeset = match next_changeset {
+                Ok(next_changeset) => next_changeset,
+                Err(iter_error) => {
+                    return Err(AggregateChangesetsError {
+                        changeset,
+                        iter_error,
+                    })
+                }
+            };
+            match &mut changeset {
+                Some(changeset) => changeset.append(next_changeset),
+                changeset => *changeset = Some(next_changeset),
             }
-            Ok(())
-        })();
-
-        (changeset, result)
+        }
+        Ok(changeset)
     }
 
     /// Append a new changeset to the file and truncate the file to the end of the appended
@@ -148,7 +168,7 @@ where
         bincode_options()
             .serialize_into(&mut self.db_file, changeset)
             .map_err(|e| match *e {
-                bincode::ErrorKind::Io(inner) => inner,
+                bincode::ErrorKind::Io(error) => error,
                 unexpected_err => panic!("unexpected bincode error: {}", unexpected_err),
             })?;
 
@@ -162,12 +182,31 @@ where
     }
 }
 
+/// Error type for [`Store::aggregate_changesets`].
+#[derive(Debug)]
+pub struct AggregateChangesetsError<C> {
+    /// The partially-aggregated changeset.
+    pub changeset: Option<C>,
+
+    /// The error returned by [`EntryIter`].
+    pub iter_error: IterError,
+}
+
+impl<C> std::fmt::Display for AggregateChangesetsError<C> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.iter_error, f)
+    }
+}
+
+impl<C: fmt::Debug> std::error::Error for AggregateChangesetsError<C> {}
+
 #[cfg(test)]
 mod test {
     use super::*;
 
     use bincode::DefaultOptions;
     use std::{
+        collections::BTreeSet,
         io::{Read, Write},
         vec::Vec,
     };
@@ -177,10 +216,44 @@ mod test {
     const TEST_MAGIC_BYTES: [u8; TEST_MAGIC_BYTES_LEN] =
         [98, 100, 107, 102, 115, 49, 49, 49, 49, 49, 49, 49];
 
-    type TestChangeSet = Vec<String>;
+    type TestChangeSet = BTreeSet<String>;
 
-    #[derive(Debug)]
-    struct TestTracker;
+    /// Check behavior of [`Store::create_new`] and [`Store::open`].
+    #[test]
+    fn construct_store() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let file_path = temp_dir.path().join("db_file");
+        let _ = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+            .expect_err("must not open as file does not exist yet");
+        let _ = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path)
+            .expect("must create file");
+        // cannot create new as file already exists
+        let _ = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path)
+            .expect_err("must fail as file already exists now");
+        let _ = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+            .expect("must open as file exists now");
+    }
+
+    #[test]
+    fn open_or_create_new() {
+        let temp_dir = tempfile::tempdir().unwrap();
+        let file_path = temp_dir.path().join("db_file");
+        let changeset = BTreeSet::from(["hello".to_string(), "world".to_string()]);
+
+        {
+            let mut db = Store::<TestChangeSet>::open_or_create_new(&TEST_MAGIC_BYTES, &file_path)
+                .expect("must create");
+            assert!(file_path.exists());
+            db.append_changeset(&changeset).expect("must succeed");
+        }
+
+        {
+            let mut db = Store::<TestChangeSet>::open_or_create_new(&TEST_MAGIC_BYTES, &file_path)
+                .expect("must recover");
+            let recovered_changeset = db.aggregate_changesets().expect("must succeed");
+            assert_eq!(recovered_changeset, Some(changeset));
+        }
+    }
 
     #[test]
     fn new_fails_if_file_is_too_short() {
@@ -188,7 +261,7 @@ mod test {
         file.write_all(&TEST_MAGIC_BYTES[..TEST_MAGIC_BYTES_LEN - 1])
             .expect("should write");
 
-        match Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap()) {
+        match Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()) {
             Err(FileError::Io(e)) => assert_eq!(e.kind(), std::io::ErrorKind::UnexpectedEof),
             unexpected => panic!("unexpected result: {:?}", unexpected),
         };
@@ -202,7 +275,7 @@ mod test {
         file.write_all(invalid_magic_bytes.as_bytes())
             .expect("should write");
 
-        match Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap()) {
+        match Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()) {
             Err(FileError::InvalidMagicBytes { got, .. }) => {
                 assert_eq!(got, invalid_magic_bytes.as_bytes())
             }
@@ -216,13 +289,13 @@ mod test {
         let mut data = [255_u8; 2000];
         data[..TEST_MAGIC_BYTES_LEN].copy_from_slice(&TEST_MAGIC_BYTES);
 
-        let changeset = vec!["one".into(), "two".into(), "three!".into()];
+        let changeset = TestChangeSet::from(["one".into(), "two".into(), "three!".into()]);
 
         let mut file = NamedTempFile::new().unwrap();
         file.write_all(&data).expect("should write");
 
-        let mut store = Store::<TestChangeSet>::new(&TEST_MAGIC_BYTES, file.reopen().unwrap())
-            .expect("should open");
+        let mut store =
+            Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, file.path()).expect("should open");
         match store.iter_changesets().next() {
             Some(Err(IterError::Bincode(_))) => {}
             unexpected_res => panic!("unexpected result: {:?}", unexpected_res),
@@ -252,4 +325,119 @@ mod test {
 
         assert_eq!(got_bytes, expected_bytes);
     }
+
+    #[test]
+    fn last_write_is_short() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let changesets = [
+            TestChangeSet::from(["1".into()]),
+            TestChangeSet::from(["2".into(), "3".into()]),
+            TestChangeSet::from(["4".into(), "5".into(), "6".into()]),
+        ];
+        let last_changeset = TestChangeSet::from(["7".into(), "8".into(), "9".into()]);
+        let last_changeset_bytes = bincode_options().serialize(&last_changeset).unwrap();
+
+        for short_write_len in 1..last_changeset_bytes.len() - 1 {
+            let file_path = temp_dir.path().join(format!("{}.dat", short_write_len));
+            println!("Test file: {:?}", file_path);
+
+            // simulate creating a file, writing data where the last write is incomplete
+            {
+                let mut db =
+                    Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                for changeset in &changesets {
+                    db.append_changeset(changeset).unwrap();
+                }
+                // this is the incomplete write
+                db.db_file
+                    .write_all(&last_changeset_bytes[..short_write_len])
+                    .unwrap();
+            }
+
+            // load file again and aggregate changesets
+            // write the last changeset again (this time it succeeds)
+            {
+                let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                let err = db
+                    .aggregate_changesets()
+                    .expect_err("should return error as last read is short");
+                assert_eq!(
+                    err.changeset,
+                    changesets.iter().cloned().reduce(|mut acc, cs| {
+                        Append::append(&mut acc, cs);
+                        acc
+                    }),
+                    "should recover all changesets that are written in full",
+                );
+                db.db_file.write_all(&last_changeset_bytes).unwrap();
+            }
+
+            // load file again - this time we should successfully aggregate all changesets
+            {
+                let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+                let aggregated_changesets = db
+                    .aggregate_changesets()
+                    .expect("aggregating all changesets should succeed");
+                assert_eq!(
+                    aggregated_changesets,
+                    changesets
+                        .iter()
+                        .cloned()
+                        .chain(core::iter::once(last_changeset.clone()))
+                        .reduce(|mut acc, cs| {
+                            Append::append(&mut acc, cs);
+                            acc
+                        }),
+                    "should recover all changesets",
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn write_after_short_read() {
+        let temp_dir = tempfile::tempdir().unwrap();
+
+        let changesets = (0..20)
+            .map(|n| TestChangeSet::from([format!("{}", n)]))
+            .collect::<Vec<_>>();
+        let last_changeset = TestChangeSet::from(["last".into()]);
+
+        for read_count in 0..changesets.len() {
+            let file_path = temp_dir.path().join(format!("{}.dat", read_count));
+            println!("Test file: {:?}", file_path);
+
+            // First, we create the file with all the changesets!
+            let mut db = Store::<TestChangeSet>::create_new(&TEST_MAGIC_BYTES, &file_path).unwrap();
+            for changeset in &changesets {
+                db.append_changeset(changeset).unwrap();
+            }
+            drop(db);
+
+            // We re-open the file and read `read_count` number of changesets.
+            let mut db = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path).unwrap();
+            let mut exp_aggregation = db
+                .iter_changesets()
+                .take(read_count)
+                .map(|r| r.expect("must read valid changeset"))
+                .fold(TestChangeSet::default(), |mut acc, v| {
+                    Append::append(&mut acc, v);
+                    acc
+                });
+            // We write after a short read.
+            db.append_changeset(&last_changeset)
+                .expect("last write must succeed");
+            Append::append(&mut exp_aggregation, last_changeset.clone());
+            drop(db);
+
+            // We open the file again and check whether aggregate changeset is expected.
+            let aggregation = Store::<TestChangeSet>::open(&TEST_MAGIC_BYTES, &file_path)
+                .unwrap()
+                .aggregate_changesets()
+                .expect("must aggregate changesets")
+                .unwrap_or_default();
+            assert_eq!(aggregation, exp_aggregation);
+        }
+    }
 }

crates/file_store/tests/test_file_store.rs

@@ -1 +0,0 @@
-

crates/hwi/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "bdk_hwi"
+version = "0.3.0"
+edition = "2021"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+description = "Utilities to use bdk with hardware wallets"
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+[dependencies]
+bdk_wallet = { path = "../wallet", version = "1.0.0-alpha.13" }
+hwi = { version = "0.9.0", features = [ "miniscript"] }

crates/hwi/README.md

@@ -0,0 +1,3 @@
+# BDK HWI Signer
+
+This crate contains `HWISigner`, an implementation of a `TransactionSigner` to be used with hardware wallets.

crates/hwi/src/lib.rs

@@ -0,0 +1,41 @@
+//! HWI Signer
+//!
+//! This crate contains HWISigner, an implementation of a [`TransactionSigner`] to be
+//! used with hardware wallets.
+//! ```no_run
+//! # use bdk_wallet::bitcoin::Network;
+//! # use bdk_wallet::signer::SignerOrdering;
+//! # use bdk_hwi::HWISigner;
+//! # use bdk_wallet::{KeychainKind, SignOptions, Wallet};
+//! # use hwi::HWIClient;
+//! # use std::sync::Arc;
+//! #
+//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! let mut devices = HWIClient::enumerate()?;
+//! if devices.is_empty() {
+//!     panic!("No devices found!");
+//! }
+//! let first_device = devices.remove(0)?;
+//! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
+//!
+//! # let mut wallet = Wallet::new(
+//! #     "",
+//! #     "",
+//! #     Network::Testnet,
+//! # )?;
+//! #
+//! // Adding the hardware signer to the BDK wallet
+//! wallet.add_signer(
+//!     KeychainKind::External,
+//!     SignerOrdering(200),
+//!     Arc::new(custom_signer),
+//! );
+//!
+//! # Ok(())
+//! # }
+//! ```
+//!
+//! [`TransactionSigner`]: bdk_wallet::wallet::signer::TransactionSigner
+
+mod signer;
+pub use signer::*;

crates/hwi/src/signer.rs

@@ -0,0 +1,94 @@
+use bdk_wallet::bitcoin::bip32::Fingerprint;
+use bdk_wallet::bitcoin::secp256k1::{All, Secp256k1};
+use bdk_wallet::bitcoin::Psbt;
+
+use hwi::error::Error;
+use hwi::types::{HWIChain, HWIDevice};
+use hwi::HWIClient;
+
+use bdk_wallet::signer::{SignerCommon, SignerError, SignerId, TransactionSigner};
+
+#[derive(Debug)]
+/// Custom signer for Hardware Wallets
+///
+/// This ignores `sign_options` and leaves the decisions up to the hardware wallet.
+pub struct HWISigner {
+    fingerprint: Fingerprint,
+    client: HWIClient,
+}
+
+impl HWISigner {
+    /// Create a instance from the specified device and chain
+    pub fn from_device(device: &HWIDevice, chain: HWIChain) -> Result<HWISigner, Error> {
+        let client = HWIClient::get_client(device, false, chain)?;
+        Ok(HWISigner {
+            fingerprint: device.fingerprint,
+            client,
+        })
+    }
+}
+
+impl SignerCommon for HWISigner {
+    fn id(&self, _secp: &Secp256k1<All>) -> SignerId {
+        SignerId::Fingerprint(self.fingerprint)
+    }
+}
+
+impl TransactionSigner for HWISigner {
+    fn sign_transaction(
+        &self,
+        psbt: &mut Psbt,
+        _sign_options: &bdk_wallet::SignOptions,
+        _secp: &Secp256k1<All>,
+    ) -> Result<(), SignerError> {
+        psbt.combine(
+            self.client
+                .sign_tx(psbt)
+                .map_err(|e| {
+                    SignerError::External(format!("While signing with hardware wallet: {}", e))
+                })?
+                .psbt,
+        )
+        .expect("Failed to combine HW signed psbt with passed PSBT");
+        Ok(())
+    }
+}
+
+// TODO: re-enable this once we have the `get_funded_wallet` test util
+// #[cfg(test)]
+// mod tests {
+//     #[test]
+//     fn test_hardware_signer() {
+//         use std::sync::Arc;
+//
+//         use bdk_wallet::tests::get_funded_wallet;
+//         use bdk_wallet::signer::SignerOrdering;
+//         use bdk_wallet::bitcoin::Network;
+//         use crate::HWISigner;
+//         use hwi::HWIClient;
+//
+//         let mut devices = HWIClient::enumerate().unwrap();
+//         if devices.is_empty() {
+//             panic!("No devices found!");
+//         }
+//         let device = devices.remove(0).unwrap();
+//         let client = HWIClient::get_client(&device, true, Network::Regtest.into()).unwrap();
+//         let descriptors = client.get_descriptors::<String>(None).unwrap();
+//         let custom_signer = HWISigner::from_device(&device, Network::Regtest.into()).unwrap();
+//
+//         let (mut wallet, _) = get_funded_wallet(&descriptors.internal[0]);
+//         wallet.add_signer(
+//             bdk_wallet::KeychainKind::External,
+//             SignerOrdering(200),
+//             Arc::new(custom_signer),
+//         );
+//
+//         let addr = wallet.get_address(bdk_wallet::wallet::AddressIndex::LastUnused);
+//         let mut builder = wallet.build_tx();
+//         builder.drain_to(addr.script_pubkey()).drain_wallet();
+//         let (mut psbt, _) = builder.finish().unwrap();
+//
+//         let finalized = wallet.sign(&mut psbt, Default::default()).unwrap();
+//         assert!(finalized);
+//     }
+// }

crates/sqlite/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "bdk_sqlite"
+version = "0.2.0"
+edition = "2021"
+license = "MIT OR Apache-2.0"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_sqlite"
+description = "A simple SQLite relational database client for persisting bdk_chain data."
+keywords = ["bitcoin", "persist", "persistence", "bdk", "sqlite"]
+authors = ["Bitcoin Dev Kit Developers"]
+readme = "README.md"
+
+[dependencies]
+bdk_chain = { path = "../chain", version = "0.16.0", features = ["serde", "miniscript"] }
+rusqlite = { version = "0.31.0", features = ["bundled"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"

crates/sqlite/README.md

@@ -0,0 +1,8 @@
+# BDK SQLite
+
+This is a simple [SQLite] relational database client for persisting [`bdk_chain`] changesets.
+
+The main structure is `Store` which persists `CombinedChangeSet` data into a SQLite database file.
+
+[`bdk_chain`]:https://docs.rs/bdk_chain/latest/bdk_chain/
+[SQLite]: https://www.sqlite.org/index.html

crates/sqlite/schema/schema_0.sql

@@ -0,0 +1,69 @@
+-- schema version control
+CREATE TABLE version
+(
+    version INTEGER
+) STRICT;
+INSERT INTO version
+VALUES (1);
+
+-- network is the valid network for all other table data
+CREATE TABLE network
+(
+    name TEXT UNIQUE NOT NULL
+) STRICT;
+
+-- keychain is the json serialized keychain structure as JSONB,
+-- descriptor is the complete descriptor string,
+-- descriptor_id is a sha256::Hash id of the descriptor string w/o the checksum,
+-- last revealed index is a u32
+CREATE TABLE keychain
+(
+    keychain      BLOB PRIMARY KEY NOT NULL,
+    descriptor    TEXT             NOT NULL,
+    descriptor_id BLOB             NOT NULL,
+    last_revealed INTEGER
+) STRICT;
+
+-- hash is block hash hex string,
+-- block height is a u32,
+CREATE TABLE block
+(
+    hash   TEXT PRIMARY KEY NOT NULL,
+    height INTEGER          NOT NULL
+) STRICT;
+
+-- txid is transaction hash hex string (reversed)
+-- whole_tx is a consensus encoded transaction,
+-- last seen is a u64 unix epoch seconds
+CREATE TABLE tx
+(
+    txid      TEXT PRIMARY KEY NOT NULL,
+    whole_tx  BLOB,
+    last_seen INTEGER
+) STRICT;
+
+-- Outpoint txid hash hex string (reversed)
+-- Outpoint vout
+-- TxOut value as SATs
+-- TxOut script consensus encoded
+CREATE TABLE txout
+(
+    txid   TEXT    NOT NULL,
+    vout   INTEGER NOT NULL,
+    value  INTEGER NOT NULL,
+    script BLOB    NOT NULL,
+    PRIMARY KEY (txid, vout)
+) STRICT;
+
+-- join table between anchor and tx
+-- block hash hex string
+-- anchor is a json serialized Anchor structure as JSONB,
+-- txid is transaction hash hex string (reversed)
+CREATE TABLE anchor_tx
+(
+    block_hash          TEXT NOT NULL,
+    anchor              BLOB NOT NULL,
+    txid                TEXT NOT NULL REFERENCES tx (txid),
+    UNIQUE (anchor, txid),
+    FOREIGN KEY (block_hash) REFERENCES block(hash)
+) STRICT;

crates/sqlite/src/lib.rs

@@ -0,0 +1,34 @@
+#![doc = include_str!("../README.md")]
+// only enables the `doc_cfg` feature when the `docsrs` configuration attribute is defined
+#![cfg_attr(docsrs, feature(doc_cfg))]
+
+mod schema;
+mod store;
+
+use bdk_chain::bitcoin::Network;
+pub use rusqlite;
+pub use store::Store;
+
+/// Error that occurs while reading or writing change sets with the SQLite database.
+#[derive(Debug)]
+pub enum Error {
+    /// Invalid network, cannot change the one already stored in the database.
+    Network { expected: Network, given: Network },
+    /// SQLite error.
+    Sqlite(rusqlite::Error),
+}
+
+impl core::fmt::Display for Error {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        match self {
+            Self::Network { expected, given } => write!(
+                f,
+                "network error trying to read or write change set, expected {}, given {}",
+                expected, given
+            ),
+            Self::Sqlite(e) => write!(f, "sqlite error reading or writing changeset: {}", e),
+        }
+    }
+}
+
+impl std::error::Error for Error {}

crates/sqlite/src/schema.rs

@@ -0,0 +1,96 @@
+use crate::Store;
+use rusqlite::{named_params, Connection, Error};
+
+const SCHEMA_0: &str = include_str!("../schema/schema_0.sql");
+const MIGRATIONS: &[&str] = &[SCHEMA_0];
+
+/// Schema migration related functions.
+impl<K, A> Store<K, A> {
+    /// Migrate sqlite db schema to latest version.
+    pub(crate) fn migrate(conn: &mut Connection) -> Result<(), Error> {
+        let stmts = &MIGRATIONS
+            .iter()
+            .flat_map(|stmt| {
+                // remove comment lines
+                let s = stmt
+                    .split('\n')
+                    .filter(|l| !l.starts_with("--") && !l.is_empty())
+                    .collect::<Vec<_>>()
+                    .join(" ");
+                // split into statements
+                s.split(';')
+                    // remove extra spaces
+                    .map(|s| {
+                        s.trim()
+                            .split(' ')
+                            .filter(|s| !s.is_empty())
+                            .collect::<Vec<_>>()
+                            .join(" ")
+                    })
+                    .collect::<Vec<_>>()
+            })
+            // remove empty statements
+            .filter(|s| !s.is_empty())
+            .collect::<Vec<String>>();
+
+        let version = Self::get_schema_version(conn)?;
+        let stmts = &stmts[(version as usize)..];
+
+        // begin transaction, all migration statements and new schema version commit or rollback
+        let tx = conn.transaction()?;
+
+        // execute every statement and return `Some` new schema version
+        // if execution fails, return `Error::Rusqlite`
+        // if no statements executed returns `None`
+        let new_version = stmts
+            .iter()
+            .enumerate()
+            .map(|version_stmt| {
+                tx.execute(version_stmt.1.as_str(), [])
+                    // map result value to next migration version
+                    .map(|_| version_stmt.0 as i32 + version + 1)
+            })
+            .last()
+            .transpose()?;
+
+        // if `Some` new statement version, set new schema version
+        if let Some(version) = new_version {
+            Self::set_schema_version(&tx, version)?;
+        }
+
+        // commit transaction
+        tx.commit()?;
+        Ok(())
+    }
+
+    fn get_schema_version(conn: &Connection) -> rusqlite::Result<i32> {
+        let statement = conn.prepare_cached("SELECT version FROM version");
+        match statement {
+            Err(Error::SqliteFailure(e, Some(msg))) => {
+                if msg == "no such table: version" {
+                    Ok(0)
+                } else {
+                    Err(Error::SqliteFailure(e, Some(msg)))
+                }
+            }
+            Ok(mut stmt) => {
+                let mut rows = stmt.query([])?;
+                match rows.next()? {
+                    Some(row) => {
+                        let version: i32 = row.get(0)?;
+                        Ok(version)
+                    }
+                    None => Ok(0),
+                }
+            }
+            _ => Ok(0),
+        }
+    }
+
+    fn set_schema_version(conn: &Connection, version: i32) -> rusqlite::Result<usize> {
+        conn.execute(
+            "UPDATE version SET version=:version",
+            named_params! {":version": version},
+        )
+    }
+}

crates/sqlite/src/store.rs

@@ -0,0 +1,758 @@
+use bdk_chain::bitcoin::consensus::{deserialize, serialize};
+use bdk_chain::bitcoin::hashes::Hash;
+use bdk_chain::bitcoin::{Amount, Network, OutPoint, ScriptBuf, Transaction, TxOut};
+use bdk_chain::bitcoin::{BlockHash, Txid};
+use bdk_chain::miniscript::descriptor::{Descriptor, DescriptorPublicKey};
+use rusqlite::{named_params, Connection};
+use serde::{Deserialize, Serialize};
+use std::collections::{BTreeMap, BTreeSet};
+use std::fmt::Debug;
+use std::marker::PhantomData;
+use std::str::FromStr;
+use std::sync::{Arc, Mutex};
+
+use crate::Error;
+use bdk_chain::CombinedChangeSet;
+use bdk_chain::{
+    indexed_tx_graph, keychain, local_chain, tx_graph, Anchor, Append, DescriptorExt, DescriptorId,
+};
+
+/// Persists data in to a relational schema based [SQLite] database file.
+///
+/// The changesets loaded or stored represent changes to keychain and blockchain data.
+///
+/// [SQLite]: https://www.sqlite.org/index.html
+pub struct Store<K, A> {
+    // A rusqlite connection to the SQLite database. Uses a Mutex for thread safety.
+    conn: Mutex<Connection>,
+    keychain_marker: PhantomData<K>,
+    anchor_marker: PhantomData<A>,
+}
+
+impl<K, A> Debug for Store<K, A> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        Debug::fmt(&self.conn, f)
+    }
+}
+
+impl<K, A> Store<K, A>
+where
+    K: Ord + for<'de> Deserialize<'de> + Serialize + Send,
+    A: Anchor + for<'de> Deserialize<'de> + Serialize + Send,
+{
+    /// Creates a new store from a [`Connection`].
+    pub fn new(mut conn: Connection) -> Result<Self, rusqlite::Error> {
+        Self::migrate(&mut conn)?;
+
+        Ok(Self {
+            conn: Mutex::new(conn),
+            keychain_marker: Default::default(),
+            anchor_marker: Default::default(),
+        })
+    }
+
+    pub(crate) fn db_transaction(&mut self) -> Result<rusqlite::Transaction, Error> {
+        let connection = self.conn.get_mut().expect("unlocked connection mutex");
+        connection.transaction().map_err(Error::Sqlite)
+    }
+}
+
+/// Network table related functions.
+impl<K, A> Store<K, A> {
+    /// Insert [`Network`] for which all other tables data is valid.
+    ///
+    /// Error if trying to insert different network value.
+    fn insert_network(
+        current_network: &Option<Network>,
+        db_transaction: &rusqlite::Transaction,
+        network_changeset: &Option<Network>,
+    ) -> Result<(), Error> {
+        if let Some(network) = network_changeset {
+            match current_network {
+                // if no network change do nothing
+                Some(current_network) if current_network == network => Ok(()),
+                // if new network not the same as current, error
+                Some(current_network) => Err(Error::Network {
+                    expected: *current_network,
+                    given: *network,
+                }),
+                // insert network if none exists
+                None => {
+                    let insert_network_stmt = &mut db_transaction
+                        .prepare_cached("INSERT INTO network (name) VALUES (:name)")
+                        .expect("insert network statement");
+                    let name = network.to_string();
+                    insert_network_stmt
+                        .execute(named_params! {":name": name })
+                        .map_err(Error::Sqlite)?;
+                    Ok(())
+                }
+            }
+        } else {
+            Ok(())
+        }
+    }
+
+    /// Select the valid [`Network`] for this database, or `None` if not set.
+    fn select_network(db_transaction: &rusqlite::Transaction) -> Result<Option<Network>, Error> {
+        let mut select_network_stmt = db_transaction
+            .prepare_cached("SELECT name FROM network WHERE rowid = 1")
+            .expect("select network statement");
+
+        let network = select_network_stmt
+            .query_row([], |row| {
+                let network = row.get_unwrap::<usize, String>(0);
+                let network = Network::from_str(network.as_str()).expect("valid network");
+                Ok(network)
+            })
+            .map_err(Error::Sqlite);
+        match network {
+            Ok(network) => Ok(Some(network)),
+            Err(Error::Sqlite(rusqlite::Error::QueryReturnedNoRows)) => Ok(None),
+            Err(e) => Err(e),
+        }
+    }
+}
+
+/// Block table related functions.
+impl<K, A> Store<K, A> {
+    /// Insert or delete local chain blocks.
+    ///
+    /// Error if trying to insert existing block hash.
+    fn insert_or_delete_blocks(
+        db_transaction: &rusqlite::Transaction,
+        chain_changeset: &local_chain::ChangeSet,
+    ) -> Result<(), Error> {
+        for (height, hash) in chain_changeset.iter() {
+            match hash {
+                // add new hash at height
+                Some(hash) => {
+                    let insert_block_stmt = &mut db_transaction
+                        .prepare_cached("INSERT INTO block (hash, height) VALUES (:hash, :height)")
+                        .expect("insert block statement");
+                    let hash = hash.to_string();
+                    insert_block_stmt
+                        .execute(named_params! {":hash": hash, ":height": height })
+                        .map_err(Error::Sqlite)?;
+                }
+                // delete block at height
+                None => {
+                    let delete_block_stmt = &mut db_transaction
+                        .prepare_cached("DELETE FROM block WHERE height IS :height")
+                        .expect("delete block statement");
+                    delete_block_stmt
+                        .execute(named_params! {":height": height })
+                        .map_err(Error::Sqlite)?;
+                }
+            }
+        }
+
+        Ok(())
+    }
+
+    /// Select all blocks.
+    fn select_blocks(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeMap<u32, Option<BlockHash>>, Error> {
+        let mut select_blocks_stmt = db_transaction
+            .prepare_cached("SELECT height, hash FROM block")
+            .expect("select blocks statement");
+
+        let blocks = select_blocks_stmt
+            .query_map([], |row| {
+                let height = row.get_unwrap::<usize, u32>(0);
+                let hash = row.get_unwrap::<usize, String>(1);
+                let hash = Some(BlockHash::from_str(hash.as_str()).expect("block hash"));
+                Ok((height, hash))
+            })
+            .map_err(Error::Sqlite)?;
+        blocks
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+}
+
+/// Keychain table related functions.
+///
+/// The keychain objects are stored as [`JSONB`] data.
+/// [`JSONB`]: https://sqlite.org/json1.html#jsonb
+impl<K, A> Store<K, A>
+where
+    K: Ord + for<'de> Deserialize<'de> + Serialize + Send,
+    A: Anchor + Send,
+{
+    /// Insert keychain with descriptor and last active index.
+    ///
+    /// If keychain exists only update last active index.
+    fn insert_keychains(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        let keychain_changeset = &tx_graph_changeset.indexer;
+        for (keychain, descriptor) in keychain_changeset.keychains_added.iter() {
+            let insert_keychain_stmt = &mut db_transaction
+                .prepare_cached("INSERT INTO keychain (keychain, descriptor, descriptor_id) VALUES (jsonb(:keychain), :descriptor, :descriptor_id)")
+                .expect("insert keychain statement");
+            let keychain_json = serde_json::to_string(keychain).expect("keychain json");
+            let descriptor_id = descriptor.descriptor_id().to_byte_array();
+            let descriptor = descriptor.to_string();
+            insert_keychain_stmt.execute(named_params! {":keychain": keychain_json, ":descriptor": descriptor, ":descriptor_id": descriptor_id })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+
+    /// Update descriptor last revealed index.
+    fn update_last_revealed(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        let keychain_changeset = &tx_graph_changeset.indexer;
+        for (descriptor_id, last_revealed) in keychain_changeset.last_revealed.iter() {
+            let update_last_revealed_stmt = &mut db_transaction
+                .prepare_cached(
+                    "UPDATE keychain SET last_revealed = :last_revealed
+                              WHERE descriptor_id = :descriptor_id",
+                )
+                .expect("update last revealed statement");
+            let descriptor_id = descriptor_id.to_byte_array();
+            update_last_revealed_stmt.execute(named_params! {":descriptor_id": descriptor_id, ":last_revealed": * last_revealed })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+
+    /// Select keychains added.
+    fn select_keychains(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeMap<K, Descriptor<DescriptorPublicKey>>, Error> {
+        let mut select_keychains_added_stmt = db_transaction
+            .prepare_cached("SELECT json(keychain), descriptor FROM keychain")
+            .expect("select keychains statement");
+
+        let keychains = select_keychains_added_stmt
+            .query_map([], |row| {
+                let keychain = row.get_unwrap::<usize, String>(0);
+                let keychain = serde_json::from_str::<K>(keychain.as_str()).expect("keychain");
+                let descriptor = row.get_unwrap::<usize, String>(1);
+                let descriptor = Descriptor::from_str(descriptor.as_str()).expect("descriptor");
+                Ok((keychain, descriptor))
+            })
+            .map_err(Error::Sqlite)?;
+        keychains
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+
+    /// Select descriptor last revealed indexes.
+    fn select_last_revealed(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeMap<DescriptorId, u32>, Error> {
+        let mut select_last_revealed_stmt = db_transaction
+            .prepare_cached(
+                "SELECT descriptor, last_revealed FROM keychain WHERE last_revealed IS NOT NULL",
+            )
+            .expect("select last revealed statement");
+
+        let last_revealed = select_last_revealed_stmt
+            .query_map([], |row| {
+                let descriptor = row.get_unwrap::<usize, String>(0);
+                let descriptor = Descriptor::from_str(descriptor.as_str()).expect("descriptor");
+                let descriptor_id = descriptor.descriptor_id();
+                let last_revealed = row.get_unwrap::<usize, u32>(1);
+                Ok((descriptor_id, last_revealed))
+            })
+            .map_err(Error::Sqlite)?;
+        last_revealed
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+}
+
+/// Tx (transaction) and txout (transaction output) table related functions.
+impl<K, A> Store<K, A> {
+    /// Insert transactions.
+    ///
+    /// Error if trying to insert existing txid.
+    fn insert_txs(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        for tx in tx_graph_changeset.graph.txs.iter() {
+            let insert_tx_stmt = &mut db_transaction
+                .prepare_cached("INSERT INTO tx (txid, whole_tx) VALUES (:txid, :whole_tx) ON CONFLICT (txid) DO UPDATE SET whole_tx = :whole_tx WHERE txid = :txid")
+                .expect("insert or update tx whole_tx statement");
+            let txid = tx.compute_txid().to_string();
+            let whole_tx = serialize(&tx);
+            insert_tx_stmt
+                .execute(named_params! {":txid": txid, ":whole_tx": whole_tx })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+
+    /// Select all transactions.
+    fn select_txs(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeSet<Arc<Transaction>>, Error> {
+        let mut select_tx_stmt = db_transaction
+            .prepare_cached("SELECT whole_tx FROM tx WHERE whole_tx IS NOT NULL")
+            .expect("select tx statement");
+
+        let txs = select_tx_stmt
+            .query_map([], |row| {
+                let whole_tx = row.get_unwrap::<usize, Vec<u8>>(0);
+                let whole_tx: Transaction = deserialize(&whole_tx).expect("transaction");
+                Ok(Arc::new(whole_tx))
+            })
+            .map_err(Error::Sqlite)?;
+
+        txs.into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+
+    /// Select all transactions with last_seen values.
+    fn select_last_seen(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeMap<Txid, u64>, Error> {
+        // load tx last_seen
+        let mut select_last_seen_stmt = db_transaction
+            .prepare_cached("SELECT txid, last_seen FROM tx WHERE last_seen IS NOT NULL")
+            .expect("select tx last seen statement");
+
+        let last_seen = select_last_seen_stmt
+            .query_map([], |row| {
+                let txid = row.get_unwrap::<usize, String>(0);
+                let txid = Txid::from_str(&txid).expect("txid");
+                let last_seen = row.get_unwrap::<usize, u64>(1);
+                Ok((txid, last_seen))
+            })
+            .map_err(Error::Sqlite)?;
+        last_seen
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+
+    /// Insert txouts.
+    ///
+    /// Error if trying to insert existing outpoint.
+    fn insert_txouts(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        for txout in tx_graph_changeset.graph.txouts.iter() {
+            let insert_txout_stmt = &mut db_transaction
+                .prepare_cached("INSERT INTO txout (txid, vout, value, script) VALUES (:txid, :vout, :value, :script)")
+                .expect("insert txout statement");
+            let txid = txout.0.txid.to_string();
+            let vout = txout.0.vout;
+            let value = txout.1.value.to_sat();
+            let script = txout.1.script_pubkey.as_bytes();
+            insert_txout_stmt.execute(named_params! {":txid": txid, ":vout": vout, ":value": value, ":script": script })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+
+    /// Select all transaction outputs.
+    fn select_txouts(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeMap<OutPoint, TxOut>, Error> {
+        // load tx outs
+        let mut select_txout_stmt = db_transaction
+            .prepare_cached("SELECT txid, vout, value, script FROM txout")
+            .expect("select txout statement");
+
+        let txouts = select_txout_stmt
+            .query_map([], |row| {
+                let txid = row.get_unwrap::<usize, String>(0);
+                let txid = Txid::from_str(&txid).expect("txid");
+                let vout = row.get_unwrap::<usize, u32>(1);
+                let outpoint = OutPoint::new(txid, vout);
+                let value = row.get_unwrap::<usize, u64>(2);
+                let script_pubkey = row.get_unwrap::<usize, Vec<u8>>(3);
+                let script_pubkey = ScriptBuf::from_bytes(script_pubkey);
+                let txout = TxOut {
+                    value: Amount::from_sat(value),
+                    script_pubkey,
+                };
+                Ok((outpoint, txout))
+            })
+            .map_err(Error::Sqlite)?;
+        txouts
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+
+    /// Update transaction last seen times.
+    fn update_last_seen(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        for tx_last_seen in tx_graph_changeset.graph.last_seen.iter() {
+            let insert_or_update_tx_stmt = &mut db_transaction
+                .prepare_cached("INSERT INTO tx (txid, last_seen) VALUES (:txid, :last_seen) ON CONFLICT (txid) DO UPDATE SET last_seen = :last_seen WHERE txid = :txid")
+                .expect("insert or update tx last_seen statement");
+            let txid = tx_last_seen.0.to_string();
+            let last_seen = *tx_last_seen.1;
+            insert_or_update_tx_stmt
+                .execute(named_params! {":txid": txid, ":last_seen": last_seen })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+}
+
+/// Anchor table related functions.
+impl<K, A> Store<K, A>
+where
+    K: Ord + for<'de> Deserialize<'de> + Serialize + Send,
+    A: Anchor + for<'de> Deserialize<'de> + Serialize + Send,
+{
+    /// Insert anchors.
+    fn insert_anchors(
+        db_transaction: &rusqlite::Transaction,
+        tx_graph_changeset: &indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>>,
+    ) -> Result<(), Error> {
+        // serde_json::to_string
+        for anchor in tx_graph_changeset.graph.anchors.iter() {
+            let insert_anchor_stmt = &mut db_transaction
+                .prepare_cached("INSERT INTO anchor_tx (block_hash, anchor, txid) VALUES (:block_hash, jsonb(:anchor), :txid)")
+                .expect("insert anchor statement");
+            let block_hash = anchor.0.anchor_block().hash.to_string();
+            let anchor_json = serde_json::to_string(&anchor.0).expect("anchor json");
+            let txid = anchor.1.to_string();
+            insert_anchor_stmt.execute(named_params! {":block_hash": block_hash, ":anchor": anchor_json, ":txid": txid })
+                .map_err(Error::Sqlite)?;
+        }
+        Ok(())
+    }
+
+    /// Select all anchors.
+    fn select_anchors(
+        db_transaction: &rusqlite::Transaction,
+    ) -> Result<BTreeSet<(A, Txid)>, Error> {
+        // serde_json::from_str
+        let mut select_anchor_stmt = db_transaction
+            .prepare_cached("SELECT block_hash, json(anchor), txid FROM anchor_tx")
+            .expect("select anchor statement");
+        let anchors = select_anchor_stmt
+            .query_map([], |row| {
+                let hash = row.get_unwrap::<usize, String>(0);
+                let hash = BlockHash::from_str(hash.as_str()).expect("block hash");
+                let anchor = row.get_unwrap::<usize, String>(1);
+                let anchor: A = serde_json::from_str(anchor.as_str()).expect("anchor");
+                // double check anchor blob block hash matches
+                assert_eq!(hash, anchor.anchor_block().hash);
+                let txid = row.get_unwrap::<usize, String>(2);
+                let txid = Txid::from_str(&txid).expect("txid");
+                Ok((anchor, txid))
+            })
+            .map_err(Error::Sqlite)?;
+        anchors
+            .into_iter()
+            .map(|row| row.map_err(Error::Sqlite))
+            .collect()
+    }
+}
+
+/// Functions to read and write all [`CombinedChangeSet`] data.
+impl<K, A> Store<K, A>
+where
+    K: Ord + for<'de> Deserialize<'de> + Serialize + Send,
+    A: Anchor + for<'de> Deserialize<'de> + Serialize + Send,
+{
+    /// Write the given `changeset` atomically.
+    pub fn write(&mut self, changeset: &CombinedChangeSet<K, A>) -> Result<(), Error> {
+        // no need to write anything if changeset is empty
+        if changeset.is_empty() {
+            return Ok(());
+        }
+
+        let db_transaction = self.db_transaction()?;
+
+        let network_changeset = &changeset.network;
+        let current_network = Self::select_network(&db_transaction)?;
+        Self::insert_network(&current_network, &db_transaction, network_changeset)?;
+
+        let chain_changeset = &changeset.chain;
+        Self::insert_or_delete_blocks(&db_transaction, chain_changeset)?;
+
+        let tx_graph_changeset = &changeset.indexed_tx_graph;
+        Self::insert_keychains(&db_transaction, tx_graph_changeset)?;
+        Self::update_last_revealed(&db_transaction, tx_graph_changeset)?;
+        Self::insert_txs(&db_transaction, tx_graph_changeset)?;
+        Self::insert_txouts(&db_transaction, tx_graph_changeset)?;
+        Self::insert_anchors(&db_transaction, tx_graph_changeset)?;
+        Self::update_last_seen(&db_transaction, tx_graph_changeset)?;
+        db_transaction.commit().map_err(Error::Sqlite)
+    }
+
+    /// Read the entire database and return the aggregate [`CombinedChangeSet`].
+    pub fn read(&mut self) -> Result<Option<CombinedChangeSet<K, A>>, Error> {
+        let db_transaction = self.db_transaction()?;
+
+        let network = Self::select_network(&db_transaction)?;
+        let chain = Self::select_blocks(&db_transaction)?;
+        let keychains_added = Self::select_keychains(&db_transaction)?;
+        let last_revealed = Self::select_last_revealed(&db_transaction)?;
+        let txs = Self::select_txs(&db_transaction)?;
+        let last_seen = Self::select_last_seen(&db_transaction)?;
+        let txouts = Self::select_txouts(&db_transaction)?;
+        let anchors = Self::select_anchors(&db_transaction)?;
+
+        let graph: tx_graph::ChangeSet<A> = tx_graph::ChangeSet {
+            txs,
+            txouts,
+            anchors,
+            last_seen,
+        };
+
+        let indexer: keychain::ChangeSet<K> = keychain::ChangeSet {
+            keychains_added,
+            last_revealed,
+        };
+
+        let indexed_tx_graph: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<K>> =
+            indexed_tx_graph::ChangeSet { graph, indexer };
+
+        if network.is_none() && chain.is_empty() && indexed_tx_graph.is_empty() {
+            Ok(None)
+        } else {
+            Ok(Some(CombinedChangeSet {
+                chain,
+                indexed_tx_graph,
+                network,
+            }))
+        }
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+    use crate::store::Append;
+    use bdk_chain::bitcoin::consensus::encode::deserialize;
+    use bdk_chain::bitcoin::constants::genesis_block;
+    use bdk_chain::bitcoin::hashes::hex::FromHex;
+    use bdk_chain::bitcoin::transaction::Transaction;
+    use bdk_chain::bitcoin::Network::Testnet;
+    use bdk_chain::bitcoin::{secp256k1, BlockHash, OutPoint};
+    use bdk_chain::miniscript::Descriptor;
+    use bdk_chain::CombinedChangeSet;
+    use bdk_chain::{
+        indexed_tx_graph, keychain, tx_graph, BlockId, ConfirmationHeightAnchor,
+        ConfirmationTimeHeightAnchor, DescriptorExt,
+    };
+    use std::str::FromStr;
+    use std::sync::Arc;
+
+    #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Hash, Debug, Serialize, Deserialize)]
+    enum Keychain {
+        External { account: u32, name: String },
+        Internal { account: u32, name: String },
+    }
+
+    #[test]
+    fn insert_and_load_aggregate_changesets_with_confirmation_time_height_anchor() {
+        let (test_changesets, agg_test_changesets) =
+            create_test_changesets(&|height, time, hash| ConfirmationTimeHeightAnchor {
+                confirmation_height: height,
+                confirmation_time: time,
+                anchor_block: (height, hash).into(),
+            });
+
+        let conn = Connection::open_in_memory().expect("in memory connection");
+        let mut store = Store::<Keychain, ConfirmationTimeHeightAnchor>::new(conn)
+            .expect("create new memory db store");
+
+        test_changesets.iter().for_each(|changeset| {
+            store.write(changeset).expect("write changeset");
+        });
+
+        let agg_changeset = store.read().expect("aggregated changeset");
+
+        assert_eq!(agg_changeset, Some(agg_test_changesets));
+    }
+
+    #[test]
+    fn insert_and_load_aggregate_changesets_with_confirmation_height_anchor() {
+        let (test_changesets, agg_test_changesets) =
+            create_test_changesets(&|height, _time, hash| ConfirmationHeightAnchor {
+                confirmation_height: height,
+                anchor_block: (height, hash).into(),
+            });
+
+        let conn = Connection::open_in_memory().expect("in memory connection");
+        let mut store = Store::<Keychain, ConfirmationHeightAnchor>::new(conn)
+            .expect("create new memory db store");
+
+        test_changesets.iter().for_each(|changeset| {
+            store.write(changeset).expect("write changeset");
+        });
+
+        let agg_changeset = store.read().expect("aggregated changeset");
+
+        assert_eq!(agg_changeset, Some(agg_test_changesets));
+    }
+
+    #[test]
+    fn insert_and_load_aggregate_changesets_with_blockid_anchor() {
+        let (test_changesets, agg_test_changesets) =
+            create_test_changesets(&|height, _time, hash| BlockId { height, hash });
+
+        let conn = Connection::open_in_memory().expect("in memory connection");
+        let mut store = Store::<Keychain, BlockId>::new(conn).expect("create new memory db store");
+
+        test_changesets.iter().for_each(|changeset| {
+            store.write(changeset).expect("write changeset");
+        });
+
+        let agg_changeset = store.read().expect("aggregated changeset");
+
+        assert_eq!(agg_changeset, Some(agg_test_changesets));
+    }
+
+    fn create_test_changesets<A: Anchor + Copy>(
+        anchor_fn: &dyn Fn(u32, u64, BlockHash) -> A,
+    ) -> (
+        Vec<CombinedChangeSet<Keychain, A>>,
+        CombinedChangeSet<Keychain, A>,
+    ) {
+        let secp = &secp256k1::Secp256k1::signing_only();
+
+        let network_changeset = Some(Testnet);
+
+        let block_hash_0: BlockHash = genesis_block(Testnet).block_hash();
+        let block_hash_1 =
+            BlockHash::from_str("00000000b873e79784647a6c82962c70d228557d24a747ea4d1b8bbe878e1206")
+                .unwrap();
+        let block_hash_2 =
+            BlockHash::from_str("000000006c02c8ea6e4ff69651f7fcde348fb9d557a06e6957b65552002a7820")
+                .unwrap();
+
+        let block_changeset = [
+            (0, Some(block_hash_0)),
+            (1, Some(block_hash_1)),
+            (2, Some(block_hash_2)),
+        ]
+        .into();
+
+        let ext_keychain = Keychain::External {
+            account: 0,
+            name: "ext test".to_string(),
+        };
+        let (ext_desc, _ext_keymap) = Descriptor::parse_descriptor(secp, "wpkh(tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy/0/*)").unwrap();
+        let ext_desc_id = ext_desc.descriptor_id();
+        let int_keychain = Keychain::Internal {
+            account: 0,
+            name: "int test".to_string(),
+        };
+        let (int_desc, _int_keymap) = Descriptor::parse_descriptor(secp, "wpkh(tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy/1/*)").unwrap();
+        let int_desc_id = int_desc.descriptor_id();
+
+        let tx0_hex = Vec::<u8>::from_hex("01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000").unwrap();
+        let tx0: Arc<Transaction> = Arc::new(deserialize(tx0_hex.as_slice()).unwrap());
+        let tx1_hex = Vec::<u8>::from_hex("010000000001010000000000000000000000000000000000000000000000000000000000000000ffffffff025151feffffff0200f2052a010000001600149243f727dd5343293eb83174324019ec16c2630f0000000000000000776a24aa21a9ede2f61c3f71d1defd3fa999dfa36953755c690689799962b48bebd836974e8cf94c4fecc7daa2490047304402205e423a8754336ca99dbe16509b877ef1bf98d008836c725005b3c787c41ebe46022047246e4467ad7cc7f1ad98662afcaf14c115e0095a227c7b05c5182591c23e7e01000120000000000000000000000000000000000000000000000000000000000000000000000000").unwrap();
+        let tx1: Arc<Transaction> = Arc::new(deserialize(tx1_hex.as_slice()).unwrap());
+        let tx2_hex = Vec::<u8>::from_hex("01000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0e0432e7494d010e062f503253482fffffffff0100f2052a010000002321038a7f6ef1c8ca0c588aa53fa860128077c9e6c11e6830f4d7ee4e763a56b7718fac00000000").unwrap();
+        let tx2: Arc<Transaction> = Arc::new(deserialize(tx2_hex.as_slice()).unwrap());
+
+        let outpoint0_0 = OutPoint::new(tx0.compute_txid(), 0);
+        let txout0_0 = tx0.output.first().unwrap().clone();
+        let outpoint1_0 = OutPoint::new(tx1.compute_txid(), 0);
+        let txout1_0 = tx1.output.first().unwrap().clone();
+
+        let anchor1 = anchor_fn(1, 1296667328, block_hash_1);
+        let anchor2 = anchor_fn(2, 1296688946, block_hash_2);
+
+        let tx_graph_changeset = tx_graph::ChangeSet::<A> {
+            txs: [tx0.clone(), tx1.clone()].into(),
+            txouts: [(outpoint0_0, txout0_0), (outpoint1_0, txout1_0)].into(),
+            anchors: [(anchor1, tx0.compute_txid()), (anchor1, tx1.compute_txid())].into(),
+            last_seen: [
+                (tx0.compute_txid(), 1598918400),
+                (tx1.compute_txid(), 1598919121),
+                (tx2.compute_txid(), 1608919121),
+            ]
+            .into(),
+        };
+
+        let keychain_changeset = keychain::ChangeSet {
+            keychains_added: [(ext_keychain, ext_desc), (int_keychain, int_desc)].into(),
+            last_revealed: [(ext_desc_id, 124), (int_desc_id, 421)].into(),
+        };
+
+        let graph_changeset: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<Keychain>> =
+            indexed_tx_graph::ChangeSet {
+                graph: tx_graph_changeset,
+                indexer: keychain_changeset,
+            };
+
+        // test changesets to write to db
+        let mut changesets = Vec::new();
+
+        changesets.push(CombinedChangeSet {
+            chain: block_changeset,
+            indexed_tx_graph: graph_changeset,
+            network: network_changeset,
+        });
+
+        // create changeset that sets the whole tx2 and updates it's lastseen where before there was only the txid and last_seen
+        let tx_graph_changeset2 = tx_graph::ChangeSet::<A> {
+            txs: [tx2.clone()].into(),
+            txouts: BTreeMap::default(),
+            anchors: BTreeSet::default(),
+            last_seen: [(tx2.compute_txid(), 1708919121)].into(),
+        };
+
+        let graph_changeset2: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<Keychain>> =
+            indexed_tx_graph::ChangeSet {
+                graph: tx_graph_changeset2,
+                indexer: keychain::ChangeSet::default(),
+            };
+
+        changesets.push(CombinedChangeSet {
+            chain: local_chain::ChangeSet::default(),
+            indexed_tx_graph: graph_changeset2,
+            network: None,
+        });
+
+        // create changeset that adds a new anchor2 for tx0 and tx1
+        let tx_graph_changeset3 = tx_graph::ChangeSet::<A> {
+            txs: BTreeSet::default(),
+            txouts: BTreeMap::default(),
+            anchors: [(anchor2, tx0.compute_txid()), (anchor2, tx1.compute_txid())].into(),
+            last_seen: BTreeMap::default(),
+        };
+
+        let graph_changeset3: indexed_tx_graph::ChangeSet<A, keychain::ChangeSet<Keychain>> =
+            indexed_tx_graph::ChangeSet {
+                graph: tx_graph_changeset3,
+                indexer: keychain::ChangeSet::default(),
+            };
+
+        changesets.push(CombinedChangeSet {
+            chain: local_chain::ChangeSet::default(),
+            indexed_tx_graph: graph_changeset3,
+            network: None,
+        });
+
+        // aggregated test changesets
+        let agg_test_changesets =
+            changesets
+                .iter()
+                .fold(CombinedChangeSet::<Keychain, A>::default(), |mut i, cs| {
+                    i.append(cs.clone());
+                    i
+                });
+
+        (changesets, agg_test_changesets)
+    }
+}

crates/testenv/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "bdk_testenv"
+version = "0.6.0"
+edition = "2021"
+rust-version = "1.63"
+homepage = "https://bitcoindevkit.org"
+repository = "https://github.com/bitcoindevkit/bdk"
+documentation = "https://docs.rs/bdk_testenv"
+description = "Testing framework for BDK chain sources."
+license = "MIT OR Apache-2.0"
+readme = "README.md"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+bdk_chain = { path = "../chain", version = "0.16", default-features = false }
+electrsd = { version = "0.28.0", features = ["bitcoind_25_0", "esplora_a33e97e1", "legacy"] }
+
+[features]
+default = ["std"]
+std = ["bdk_chain/std"]
+serde = ["bdk_chain/serde"]

crates/testenv/README.md

@@ -0,0 +1,6 @@
+# BDK TestEnv
+
+This crate sets up a regtest environment with a single [`bitcoind`] node
+connected to an [`electrs`] instance. This framework provides the infrastructure
+for testing chain source crates, e.g., [`bdk_chain`], [`bdk_electrum`],
+[`bdk_esplora`], etc.

crates/testenv/src/lib.rs

@@ -0,0 +1,304 @@
+use bdk_chain::{
+    bitcoin::{
+        address::NetworkChecked, block::Header, hash_types::TxMerkleNode, hashes::Hash,
+        secp256k1::rand::random, transaction, Address, Amount, Block, BlockHash, CompactTarget,
+        ScriptBuf, ScriptHash, Transaction, TxIn, TxOut, Txid,
+    },
+    local_chain::CheckPoint,
+    BlockId,
+};
+use bitcoincore_rpc::{
+    bitcoincore_rpc_json::{GetBlockTemplateModes, GetBlockTemplateRules},
+    RpcApi,
+};
+pub use electrsd;
+pub use electrsd::bitcoind;
+pub use electrsd::bitcoind::anyhow;
+pub use electrsd::bitcoind::bitcoincore_rpc;
+pub use electrsd::electrum_client;
+use electrsd::electrum_client::ElectrumApi;
+use std::time::Duration;
+
+/// Struct for running a regtest environment with a single `bitcoind` node with an `electrs`
+/// instance connected to it.
+pub struct TestEnv {
+    pub bitcoind: electrsd::bitcoind::BitcoinD,
+    pub electrsd: electrsd::ElectrsD,
+}
+
+impl TestEnv {
+    /// Construct a new [`TestEnv`] instance with default configurations.
+    pub fn new() -> anyhow::Result<Self> {
+        let bitcoind = match std::env::var_os("BITCOIND_EXE") {
+            Some(bitcoind_path) => electrsd::bitcoind::BitcoinD::new(bitcoind_path),
+            None => {
+                let bitcoind_exe = electrsd::bitcoind::downloaded_exe_path()
+                    .expect(
+                "you need to provide an env var BITCOIND_EXE or specify a bitcoind version feature",
+                );
+                electrsd::bitcoind::BitcoinD::with_conf(
+                    bitcoind_exe,
+                    &electrsd::bitcoind::Conf::default(),
+                )
+            }
+        }?;
+
+        let mut electrsd_conf = electrsd::Conf::default();
+        electrsd_conf.http_enabled = true;
+        let electrsd = match std::env::var_os("ELECTRS_EXE") {
+            Some(env_electrs_exe) => {
+                electrsd::ElectrsD::with_conf(env_electrs_exe, &bitcoind, &electrsd_conf)
+            }
+            None => {
+                let electrs_exe = electrsd::downloaded_exe_path()
+                    .expect("electrs version feature must be enabled");
+                electrsd::ElectrsD::with_conf(electrs_exe, &bitcoind, &electrsd_conf)
+            }
+        }?;
+
+        Ok(Self { bitcoind, electrsd })
+    }
+
+    /// Exposes the [`ElectrumApi`] calls from the Electrum client.
+    pub fn electrum_client(&self) -> &impl ElectrumApi {
+        &self.electrsd.client
+    }
+
+    /// Exposes the [`RpcApi`] calls from [`bitcoincore_rpc`].
+    pub fn rpc_client(&self) -> &impl RpcApi {
+        &self.bitcoind.client
+    }
+
+    // Reset `electrsd` so that new blocks can be seen.
+    pub fn reset_electrsd(mut self) -> anyhow::Result<Self> {
+        let mut electrsd_conf = electrsd::Conf::default();
+        electrsd_conf.http_enabled = true;
+        let electrsd = match std::env::var_os("ELECTRS_EXE") {
+            Some(env_electrs_exe) => {
+                electrsd::ElectrsD::with_conf(env_electrs_exe, &self.bitcoind, &electrsd_conf)
+            }
+            None => {
+                let electrs_exe = electrsd::downloaded_exe_path()
+                    .expect("electrs version feature must be enabled");
+                electrsd::ElectrsD::with_conf(electrs_exe, &self.bitcoind, &electrsd_conf)
+            }
+        }?;
+        self.electrsd = electrsd;
+        Ok(self)
+    }
+
+    /// Mine a number of blocks of a given size `count`, which may be specified to a given coinbase
+    /// `address`.
+    pub fn mine_blocks(
+        &self,
+        count: usize,
+        address: Option<Address>,
+    ) -> anyhow::Result<Vec<BlockHash>> {
+        let coinbase_address = match address {
+            Some(address) => address,
+            None => self
+                .bitcoind
+                .client
+                .get_new_address(None, None)?
+                .assume_checked(),
+        };
+        let block_hashes = self
+            .bitcoind
+            .client
+            .generate_to_address(count as _, &coinbase_address)?;
+        Ok(block_hashes)
+    }
+
+    /// Mine a block that is guaranteed to be empty even with transactions in the mempool.
+    pub fn mine_empty_block(&self) -> anyhow::Result<(usize, BlockHash)> {
+        let bt = self.bitcoind.client.get_block_template(
+            GetBlockTemplateModes::Template,
+            &[GetBlockTemplateRules::SegWit],
+            &[],
+        )?;
+
+        let txdata = vec![Transaction {
+            version: transaction::Version::ONE,
+            lock_time: bdk_chain::bitcoin::absolute::LockTime::from_height(0)?,
+            input: vec![TxIn {
+                previous_output: bdk_chain::bitcoin::OutPoint::default(),
+                script_sig: ScriptBuf::builder()
+                    .push_int(bt.height as _)
+                    // randomn number so that re-mining creates unique block
+                    .push_int(random())
+                    .into_script(),
+                sequence: bdk_chain::bitcoin::Sequence::default(),
+                witness: bdk_chain::bitcoin::Witness::new(),
+            }],
+            output: vec![TxOut {
+                value: Amount::ZERO,
+                script_pubkey: ScriptBuf::new_p2sh(&ScriptHash::all_zeros()),
+            }],
+        }];
+
+        let bits: [u8; 4] = bt
+            .bits
+            .clone()
+            .try_into()
+            .expect("rpc provided us with invalid bits");
+
+        let mut block = Block {
+            header: Header {
+                version: bdk_chain::bitcoin::block::Version::default(),
+                prev_blockhash: bt.previous_block_hash,
+                merkle_root: TxMerkleNode::all_zeros(),
+                time: Ord::max(bt.min_time, std::time::UNIX_EPOCH.elapsed()?.as_secs()) as u32,
+                bits: CompactTarget::from_consensus(u32::from_be_bytes(bits)),
+                nonce: 0,
+            },
+            txdata,
+        };
+
+        block.header.merkle_root = block.compute_merkle_root().expect("must compute");
+
+        for nonce in 0..=u32::MAX {
+            block.header.nonce = nonce;
+            if block.header.target().is_met_by(block.block_hash()) {
+                break;
+            }
+        }
+
+        self.bitcoind.client.submit_block(&block)?;
+        Ok((bt.height as usize, block.block_hash()))
+    }
+
+    /// This method waits for the Electrum notification indicating that a new block has been mined.
+    pub fn wait_until_electrum_sees_block(&self) -> anyhow::Result<()> {
+        self.electrsd.client.block_headers_subscribe()?;
+        let mut delay = Duration::from_millis(64);
+
+        loop {
+            self.electrsd.trigger()?;
+            self.electrsd.client.ping()?;
+            if self.electrsd.client.block_headers_pop()?.is_some() {
+                return Ok(());
+            }
+
+            if delay.as_millis() < 512 {
+                delay = delay.mul_f32(2.0);
+            }
+            std::thread::sleep(delay);
+        }
+    }
+
+    /// Invalidate a number of blocks of a given size `count`.
+    pub fn invalidate_blocks(&self, count: usize) -> anyhow::Result<()> {
+        let mut hash = self.bitcoind.client.get_best_block_hash()?;
+        for _ in 0..count {
+            let prev_hash = self
+                .bitcoind
+                .client
+                .get_block_info(&hash)?
+                .previousblockhash;
+            self.bitcoind.client.invalidate_block(&hash)?;
+            match prev_hash {
+                Some(prev_hash) => hash = prev_hash,
+                None => break,
+            }
+        }
+        Ok(())
+    }
+
+    /// Reorg a number of blocks of a given size `count`.
+    /// Refer to [`TestEnv::mine_empty_block`] for more information.
+    pub fn reorg(&self, count: usize) -> anyhow::Result<Vec<BlockHash>> {
+        let start_height = self.bitcoind.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = self.mine_blocks(count, None);
+        assert_eq!(
+            self.bitcoind.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        res
+    }
+
+    /// Reorg with a number of empty blocks of a given size `count`.
+    pub fn reorg_empty_blocks(&self, count: usize) -> anyhow::Result<Vec<(usize, BlockHash)>> {
+        let start_height = self.bitcoind.client.get_block_count()?;
+        self.invalidate_blocks(count)?;
+
+        let res = (0..count)
+            .map(|_| self.mine_empty_block())
+            .collect::<Result<Vec<_>, _>>()?;
+        assert_eq!(
+            self.bitcoind.client.get_block_count()?,
+            start_height,
+            "reorg should not result in height change"
+        );
+        Ok(res)
+    }
+
+    /// Send a tx of a given `amount` to a given `address`.
+    pub fn send(&self, address: &Address<NetworkChecked>, amount: Amount) -> anyhow::Result<Txid> {
+        let txid = self
+            .bitcoind
+            .client
+            .send_to_address(address, amount, None, None, None, None, None, None)?;
+        Ok(txid)
+    }
+
+    /// Create a checkpoint linked list of all the blocks in the chain.
+    pub fn make_checkpoint_tip(&self) -> CheckPoint {
+        CheckPoint::from_block_ids((0_u32..).map_while(|height| {
+            self.bitcoind
+                .client
+                .get_block_hash(height as u64)
+                .ok()
+                .map(|hash| BlockId { height, hash })
+        }))
+        .expect("must craft tip")
+    }
+
+    /// Get the genesis hash of the blockchain.
+    pub fn genesis_hash(&self) -> anyhow::Result<BlockHash> {
+        let hash = self.bitcoind.client.get_block_hash(0)?;
+        Ok(hash)
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use crate::TestEnv;
+    use electrsd::bitcoind::{anyhow::Result, bitcoincore_rpc::RpcApi};
+
+    /// This checks that reorgs initiated by `bitcoind` is detected by our `electrsd` instance.
+    #[test]
+    fn test_reorg_is_detected_in_electrsd() -> Result<()> {
+        let env = TestEnv::new()?;
+
+        // Mine some blocks.
+        env.mine_blocks(101, None)?;
+        env.wait_until_electrum_sees_block()?;
+        let height = env.bitcoind.client.get_block_count()?;
+        let blocks = (0..=height)
+            .map(|i| env.bitcoind.client.get_block_hash(i))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        // Perform reorg on six blocks.
+        env.reorg(6)?;
+        env.wait_until_electrum_sees_block()?;
+        let reorged_height = env.bitcoind.client.get_block_count()?;
+        let reorged_blocks = (0..=height)
+            .map(|i| env.bitcoind.client.get_block_hash(i))
+            .collect::<Result<Vec<_>, _>>()?;
+
+        assert_eq!(height, reorged_height);
+
+        // Block hashes should not be equal on the six reorged blocks.
+        for (i, (block, reorged_block)) in blocks.iter().zip(reorged_blocks.iter()).enumerate() {
+            match i <= height as usize - 6 {
+                true => assert_eq!(block, reorged_block),
+                false => assert_ne!(block, reorged_block),
+            }
+        }
+
+        Ok(())
+    }
+}

crates/wallet/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
-name = "bdk"
+name = "bdk_wallet"
 homepage = "https://bitcoindevkit.org"
-version = "1.0.0-alpha.1"
+version = "1.0.0-alpha.13"
 repository = "https://github.com/bitcoindevkit/bdk"
 documentation = "https://docs.rs/bdk"
 description = "A modern, lightweight, descriptor-based wallet library"
@@ -10,20 +10,18 @@ readme = "README.md"
 license = "MIT OR Apache-2.0"
 authors = ["Bitcoin Dev Kit Developers"]
 edition = "2021"
-rust-version = "1.57"
+rust-version = "1.63"
 
 [dependencies]
-log = "=0.4.18"
 rand = "^0.8"
-miniscript = { version = "9", features = ["serde"], default-features = false }
-bitcoin = { version = "0.29", features = ["serde", "base64", "rand"], default-features = false }
+miniscript = { version = "12.0.0", features = ["serde"], default-features = false }
+bitcoin = { version = "0.32.0", features = ["serde", "base64", "rand-std"], default-features = false }
 serde = { version = "^1.0", features = ["derive"] }
 serde_json = { version = "^1.0" }
-bdk_chain = { path = "../chain", version = "0.5.0", features = ["miniscript", "serde"], default-features = false }
+bdk_chain = { path = "../chain", version = "0.16.0", features = ["miniscript", "serde"], default-features = false }
 
 # Optional dependencies
-hwi = { version = "0.5", optional = true, features = [ "use-miniscript"] }
-bip39 = { version = "1.0.1", optional = true }
+bip39 = { version = "2.0", optional = true }
 
 [target.'cfg(target_arch = "wasm32")'.dependencies]
 getrandom = "0.2"
@@ -35,8 +33,6 @@ std = ["bitcoin/std", "miniscript/std", "bdk_chain/std"]
 compiler = ["miniscript/compiler"]
 all-keys = ["keys-bip39"]
 keys-bip39 = ["bip39"]
-hardware-signer = ["hwi"]
-test-hardware-signer = ["hardware-signer"]
 
 # This feature is used to run `cargo check` in our CI targeting wasm. It's not recommended
 # for libraries to explicitly include the "getrandom/js" feature, so we only do it when
@@ -45,10 +41,11 @@ dev-getrandom-wasm = ["getrandom/js"]
 
 [dev-dependencies]
 lazy_static = "1.4"
-env_logger = "0.7"
-# Move back to importing from rust-bitcoin once https://github.com/rust-bitcoin/rust-bitcoin/pull/1342 is released
-base64 = "^0.13"
 assert_matches = "1.5.0"
+tempfile = "3"
+bdk_sqlite = { path = "../sqlite" }
+bdk_file_store = { path = "../file_store" }
+anyhow = "1"
 
 [package.metadata.docs.rs]
 all-features = true

crates/wallet/README.md

@@ -8,25 +8,25 @@
   </p>
 
   <p>
-    <a href="https://crates.io/crates/bdk"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk.svg"/></a>
+    <a href="https://crates.io/crates/bdk_wallet"><img alt="Crate Info" src="https://img.shields.io/crates/v/bdk_wallet.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/blob/master/LICENSE"><img alt="MIT or Apache-2.0 Licensed" src="https://img.shields.io/badge/license-MIT%2FApache--2.0-blue.svg"/></a>
     <a href="https://github.com/bitcoindevkit/bdk/actions?query=workflow%3ACI"><img alt="CI Status" src="https://github.com/bitcoindevkit/bdk/workflows/CI/badge.svg"></a>
     <a href="https://coveralls.io/github/bitcoindevkit/bdk?branch=master"><img src="https://coveralls.io/repos/github/bitcoindevkit/bdk/badge.svg?branch=master"/></a>
-    <a href="https://docs.rs/bdk"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk-green"/></a>
-    <a href="https://blog.rust-lang.org/2021/12/02/Rust-1.57.0.html"><img alt="Rustc Version 1.57.0+" src="https://img.shields.io/badge/rustc-1.57.0%2B-lightgrey.svg"/></a>
+    <a href="https://docs.rs/bdk_wallet"><img alt="API Docs" src="https://img.shields.io/badge/docs.rs-bdk_wallet-green"/></a>
+    <a href="https://blog.rust-lang.org/2022/08/11/Rust-1.63.0.html"><img alt="Rustc Version 1.63.0+" src="https://img.shields.io/badge/rustc-1.63.0%2B-lightgrey.svg"/></a>
     <a href="https://discord.gg/d7NkDKm"><img alt="Chat on Discord" src="https://img.shields.io/discord/753336465005608961?logo=discord"></a>
   </p>
 
   <h4>
     <a href="https://bitcoindevkit.org">Project Homepage</a>
     <span> | </span>
-    <a href="https://docs.rs/bdk">Documentation</a>
+    <a href="https://docs.rs/bdk_wallet">Documentation</a>
   </h4>
 </div>
 
-## `bdk`
+# BDK Wallet
 
-The `bdk` crate provides the [`Wallet`](`crate::Wallet`) type which is a simple, high-level
+The `bdk_wallet` crate provides the [`Wallet`] type which is a simple, high-level
 interface built from the low-level components of [`bdk_chain`]. `Wallet` is a good starting point
 for many simple applications as well as a good demonstration of how to use the other mechanisms to
 construct a wallet. It has two keychains (external and internal) which are defined by
@@ -34,64 +34,78 @@ construct a wallet. It has two keychains (external and internal) which are defin
 chain data it also uses the descriptors to find transaction outputs owned by them. From there, you
 can create and sign transactions.
 
-For more information, see the [`Wallet`'s documentation](https://docs.rs/bdk/latest/bdk/wallet/struct.Wallet.html).
+For details about the API of `Wallet` see the [module-level documentation][`Wallet`].
 
-### Blockchain data
+## Blockchain data
 
-In order to get blockchain data for `Wallet` to consume, you have to put it into particular form.
-Right now this is [`KeychainScan`] which is defined in [`bdk_chain`].
-
-This can be created manually or from blockchain-scanning crates.
+In order to get blockchain data for `Wallet` to consume, you should configure a client from
+an available chain source. Typically you make a request to the chain source and get a response
+that the `Wallet` can use to update its view of the chain.
 
 **Blockchain Data Sources**
 
 * [`bdk_esplora`]: Grabs blockchain data from Esplora for updating BDK structures.
 * [`bdk_electrum`]: Grabs blockchain data from Electrum for updating BDK structures.
+* [`bdk_bitcoind_rpc`]: Grabs blockchain data from Bitcoin Core for updating BDK structures.
 
 **Examples**
 
-* [`example-crates/wallet_esplora`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora)
+* [`example-crates/wallet_esplora_async`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_async)
+* [`example-crates/wallet_esplora_blocking`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_esplora_blocking)
 * [`example-crates/wallet_electrum`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_electrum)
+* [`example-crates/wallet_rpc`](https://github.com/bitcoindevkit/bdk/tree/master/example-crates/wallet_rpc)
 
-### Persistence
+## Persistence
 
-To persist the `Wallet` on disk, `Wallet` needs to be constructed with a
-[`Persist`](https://docs.rs/bdk_chain/latest/bdk_chain/keychain/struct.KeychainPersist.html) implementation.
+To persist `Wallet` state data use a data store crate that reads and writes [`bdk_chain::CombinedChangeSet`].
 
 **Implementations**
 
-* [`bdk_file_store`]: a simple flat-file implementation of `Persist`.
+* [`bdk_file_store`]: Stores wallet changes in a simple flat file.
+* [`bdk_sqlite`]: Stores wallet changes in a SQLite relational database file.
 
 **Example**
 
-```rust
-use bdk::{bitcoin::Network, wallet::{AddressIndex, Wallet}};
+<!-- compile_fail because outpoint and txout are fake variables -->
+```rust,no_run
+use bdk_wallet::{bitcoin::Network, KeychainKind, wallet::{ChangeSet, Wallet}};
 
 fn main() {
-    // a type that implements `Persist`
-    let db = ();
+    // Open or create a new file store for wallet data.
+    let mut db =
+        bdk_file_store::Store::<ChangeSet>::open_or_create_new(b"magic_bytes", "/tmp/my_wallet.db")
+            .expect("create store");
 
-    let descriptor = "wpkh(tprv8ZgxMBicQKsPdy6LMhUtFHAgpocR8GC6QmwMSFpZs7h6Eziw3SpThFfczTDh5rW2krkqffa11UpX3XkeTTB2FvzZKWXqPY54Y6Rq4AQ5R8L/84'/0'/0'/0/*)";
-    let mut wallet = Wallet::new(descriptor, None, db, Network::Testnet).expect("should create");
+    // Create a wallet with initial wallet data read from the file store.
+    let descriptor = "wpkh(tprv8ZgxMBicQKsPdcAqYBpzAFwU5yxBUo88ggoBqu1qPcHUfSbKK1sKMLmC7EAk438btHQrSdu3jGGQa6PA71nvH5nkDexhLteJqkM4dQmWF9g/84'/1'/0'/0/*)";
+    let change_descriptor = "wpkh(tprv8ZgxMBicQKsPdcAqYBpzAFwU5yxBUo88ggoBqu1qPcHUfSbKK1sKMLmC7EAk438btHQrSdu3jGGQa6PA71nvH5nkDexhLteJqkM4dQmWF9g/84'/1'/0'/1/*)";
+    let changeset = db.aggregate_changesets().expect("changeset loaded");
+    let mut wallet =
+        Wallet::new_or_load(descriptor, change_descriptor, changeset, Network::Testnet)
+            .expect("create or load wallet");
 
-    // get a new address (this increments revealed derivation index)
-    println!("revealed address: {}", wallet.get_address(AddressIndex::New));
-    println!("staged changes: {:?}", wallet.staged());
-    // persist changes
-    wallet.commit().expect("must save");
+    // Get a new address to receive bitcoin.
+    let receive_address = wallet.reveal_next_address(KeychainKind::External);
+    // Persist staged wallet data changes to the file store.
+    let staged_changeset = wallet.take_staged();
+    if let Some(changeset) = staged_changeset {
+        db.append_changeset(&changeset)
+            .expect("must commit changes to database");
+    }
+    println!("Your new receive address is: {}", receive_address.address);
 }
 ```
 
 <!-- ### Sync the balance of a descriptor -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::Wallet; -->
-<!-- use bdk::blockchain::ElectrumBlockchain; -->
-<!-- use bdk::SyncOptions; -->
-<!-- use bdk::electrum_client::Client; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::Wallet; -->
+<!-- use bdk_wallet::blockchain::ElectrumBlockchain; -->
+<!-- use bdk_wallet::SyncOptions; -->
+<!-- use bdk_wallet::electrum_client::Client; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
 <!--     let blockchain = ElectrumBlockchain::from(Client::new("ssl://electrum.blockstream.info:60002")?); -->
 <!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
@@ -101,7 +115,7 @@ fn main() {
 
 <!--     wallet.sync(&blockchain, SyncOptions::default())?; -->
 
-<!--     println!("Descriptor balance: {} SAT", wallet.get_balance()?); -->
+<!--     println!("Descriptor balance: {} SAT", wallet.balance()?); -->
 
 <!--     Ok(()) -->
 <!-- } -->
@@ -109,12 +123,12 @@ fn main() {
 <!-- ### Generate a few addresses -->
 
 <!-- ```rust -->
-<!-- use bdk::Wallet; -->
-<!-- use bdk::wallet::AddressIndex::New; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bdk_wallet::Wallet; -->
+<!-- use bdk_wallet::wallet::AddressIndex::New; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -131,19 +145,19 @@ fn main() {
 <!-- ### Create a transaction -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::{FeeRate, Wallet, SyncOptions}; -->
-<!-- use bdk::blockchain::ElectrumBlockchain; -->
+<!-- use bdk_wallet::{FeeRate, Wallet, SyncOptions}; -->
+<!-- use bdk_wallet::blockchain::ElectrumBlockchain; -->
 
-<!-- use bdk::electrum_client::Client; -->
-<!-- use bdk::wallet::AddressIndex::New; -->
+<!-- use bdk_wallet::electrum_client::Client; -->
+<!-- use bdk_wallet::wallet::AddressIndex::New; -->
 
-<!-- use base64; -->
-<!-- use bdk::bitcoin::consensus::serialize; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bitcoin::base64; -->
+<!-- use bdk_wallet::bitcoin::consensus::serialize; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
 <!--     let blockchain = ElectrumBlockchain::from(Client::new("ssl://electrum.blockstream.info:60002")?); -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tpubDDYkZojQFQjht8Tm4jsS3iuEmKjTiEGjG6KnuFNKKJb5A6ZUCUZKdvLdSDWofKi4ToRCwb9poe1XdqfUnP4jaJjCB2Zwv11ZLgSbnZSNecE/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -172,14 +186,14 @@ fn main() {
 <!-- ### Sign a transaction -->
 
 <!-- ```rust,no_run -->
-<!-- use bdk::{Wallet, SignOptions}; -->
+<!-- use bdk_wallet::{Wallet, SignOptions}; -->
 
-<!-- use base64; -->
-<!-- use bdk::bitcoin::consensus::deserialize; -->
-<!-- use bdk::bitcoin::Network; -->
+<!-- use bitcoin::base64; -->
+<!-- use bdk_wallet::bitcoin::consensus::deserialize; -->
+<!-- use bdk_wallet::bitcoin::Network; -->
 
-<!-- fn main() -> Result<(), bdk::Error> { -->
-<!--     let wallet = Wallet::new_no_persist( -->
+<!-- fn main() -> Result<(), bdk_wallet::Error> { -->
+<!--     let wallet = Wallet::new( -->
 <!--         "wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/0/*)", -->
 <!--         Some("wpkh([c258d2e4/84h/1h/0h]tprv8griRPhA7342zfRyB6CqeKF8CJDXYu5pgnj1cjL1u2ngKcJha5jjTRimG82ABzJQ4MQe71CV54xfn25BbhCNfEGGJZnxvCDQCd6JkbvxW6h/1/*)"), -->
 <!--         Network::Testnet, -->
@@ -202,26 +216,27 @@ fn main() {
 cargo test
 ```
 
-## License
+# License
 
 Licensed under either of
 
- * Apache License, Version 2.0
-   ([LICENSE-APACHE](LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
- * MIT license
-   ([LICENSE-MIT](LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
+* Apache License, Version 2.0, ([LICENSE-APACHE](../../LICENSE-APACHE) or <https://www.apache.org/licenses/LICENSE-2.0>)
+* MIT license ([LICENSE-MIT](../../LICENSE-MIT) or <https://opensource.org/licenses/MIT>)
 
 at your option.
 
-## Contribution
+# Contribution
 
-Unless you explicitly state otherwise, any contribution intentionally submitted
-for inclusion in the work by you, as defined in the Apache-2.0 license, shall be
-dual licensed as above, without any additional terms or conditions.
+Unless you explicitly state otherwise, any contribution intentionally
+submitted for inclusion in the work by you, as defined in the Apache-2.0
+license, shall be dual licensed as above, without any additional terms or
+conditions.
 
+[`Wallet`]: https://docs.rs/bdk_wallet/latest/bdk_wallet/wallet/struct.Wallet.html
 [`bdk_chain`]: https://docs.rs/bdk_chain/latest
 [`bdk_file_store`]: https://docs.rs/bdk_file_store/latest
+[`bdk_sqlite`]: https://docs.rs/bdk_sqlite/latest
 [`bdk_electrum`]: https://docs.rs/bdk_electrum/latest
 [`bdk_esplora`]: https://docs.rs/bdk_esplora/latest
-[`KeychainScan`]: https://docs.rs/bdk_chain/latest/bdk_chain/keychain/struct.KeychainScan.html
+[`bdk_bitcoind_rpc`]: https://docs.rs/bdk_bitcoind_rpc/latest
 [`rust-miniscript`]: https://docs.rs/miniscript/latest/miniscript/index.html

crates/wallet/examples/compiler.rs

@@ -0,0 +1,96 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+extern crate bdk_wallet;
+extern crate bitcoin;
+extern crate miniscript;
+extern crate serde_json;
+
+use std::error::Error;
+use std::str::FromStr;
+
+use bitcoin::Network;
+use miniscript::policy::Concrete;
+use miniscript::Descriptor;
+
+use bdk_wallet::{KeychainKind, Wallet};
+
+/// Miniscript policy is a high level abstraction of spending conditions. Defined in the
+/// rust-miniscript library here  https://docs.rs/miniscript/7.0.0/miniscript/policy/index.html
+/// rust-miniscript provides a `compile()` function that can be used to compile any miniscript policy
+/// into a descriptor. This descriptor then in turn can be used in bdk a fully functioning wallet
+/// can be derived from the policy.
+///
+/// This example demonstrates the interaction between a bdk wallet and miniscript policy.
+
+fn main() -> Result<(), Box<dyn Error>> {
+    // We start with a miniscript policy string
+    let policy_str = "or(
+        10@thresh(4,
+            pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec),pk(02a27c8b850a00f67da3499b60562673dcf5fdfb82b7e17652a7ac54416812aefd),pk(03e618ec5f384d6e19ca9ebdb8e2119e5bef978285076828ce054e55c4daf473e2)
+        ),1@and(
+            older(4209713),
+            thresh(2,
+                pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),pk(033841045a531e1adf9910a6ec279589a90b3b8a904ee64ffd692bd08a8996c1aa),pk(02aebf2d10b040eb936a6f02f44ee82f8b34f5c1ccb20ff3949c2b28206b7c1068)
+            )
+        )
+    )"
+    .replace(&[' ', '\n', '\t'][..], "");
+
+    println!("Compiling policy: \n{}", policy_str);
+
+    // Parse the string as a [`Concrete`] type miniscript policy.
+    let policy = Concrete::<String>::from_str(&policy_str)?;
+
+    // Create a `wsh` type descriptor from the policy.
+    // `policy.compile()` returns the resulting miniscript from the policy.
+    let descriptor = Descriptor::new_wsh(policy.compile()?)?.to_string();
+
+    println!("Compiled into Descriptor: \n{}", descriptor);
+
+    // Do the same for another (internal) keychain
+    let policy_str = "or(
+        10@thresh(2,
+            pk(029ffbe722b147f3035c87cb1c60b9a5947dd49c774cc31e94773478711a929ac0),pk(025f05815e3a1a8a83bfbb03ce016c9a2ee31066b98f567f6227df1d76ec4bd143),pk(025625f41e4a065efc06d5019cbbd56fe8c07595af1231e7cbc03fafb87ebb71ec)
+        ),1@and(
+            pk(03deae92101c790b12653231439f27b8897264125ecb2f46f48278603102573165),
+            older(12960)
+        )
+    )"
+    .replace(&[' ', '\n', '\t'][..], "");
+
+    println!("Compiling internal policy: \n{}", policy_str);
+
+    let policy = Concrete::<String>::from_str(&policy_str)?;
+    let internal_descriptor = Descriptor::new_wsh(policy.compile()?)?.to_string();
+    println!(
+        "Compiled into internal Descriptor: \n{}",
+        internal_descriptor
+    );
+
+    // Create a new wallet from descriptors
+    let mut wallet = Wallet::new(&descriptor, &internal_descriptor, Network::Regtest)?;
+
+    println!(
+        "First derived address from the descriptor: \n{}",
+        wallet.next_unused_address(KeychainKind::External),
+    );
+
+    // BDK also has it's own `Policy` structure to represent the spending condition in a more
+    // human readable json format.
+    let spending_policy = wallet.policies(KeychainKind::External)?;
+    println!(
+        "The BDK spending policy: \n{}",
+        serde_json::to_string_pretty(&spending_policy)?
+    );
+
+    Ok(())
+}

crates/wallet/examples/mnemonic_to_descriptors.rs

@@ -6,21 +6,20 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-use bdk::bitcoin::secp256k1::Secp256k1;
-use bdk::bitcoin::util::bip32::DerivationPath;
-use bdk::bitcoin::Network;
-use bdk::descriptor;
-use bdk::descriptor::IntoWalletDescriptor;
-use bdk::keys::bip39::{Language, Mnemonic, WordCount};
-use bdk::keys::{GeneratableKey, GeneratedKey};
-use bdk::miniscript::Tap;
-use bdk::Error as BDK_Error;
-use std::error::Error;
+use anyhow::anyhow;
+use bdk_wallet::bitcoin::bip32::DerivationPath;
+use bdk_wallet::bitcoin::secp256k1::Secp256k1;
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::descriptor;
+use bdk_wallet::descriptor::IntoWalletDescriptor;
+use bdk_wallet::keys::bip39::{Language, Mnemonic, WordCount};
+use bdk_wallet::keys::{GeneratableKey, GeneratedKey};
+use bdk_wallet::miniscript::Tap;
 use std::str::FromStr;
 
 /// This example demonstrates how to generate a mnemonic phrase
 /// using BDK and use that to generate a descriptor string.
-fn main() -> Result<(), Box<dyn Error>> {
+fn main() -> Result<(), anyhow::Error> {
     let secp = Secp256k1::new();
 
     // In this example we are generating a 12 words mnemonic phrase
@@ -28,14 +27,14 @@ fn main() -> Result<(), Box<dyn Error>> {
     // using their respective `WordCount` variant.
     let mnemonic: GeneratedKey<_, Tap> =
         Mnemonic::generate((WordCount::Words12, Language::English))
-            .map_err(|_| BDK_Error::Generic("Mnemonic generation error".to_string()))?;
+            .map_err(|_| anyhow!("Mnemonic generation error"))?;
 
     println!("Mnemonic phrase: {}", *mnemonic);
     let mnemonic_with_passphrase = (mnemonic, None);
 
     // define external and internal derivation key path
-    let external_path = DerivationPath::from_str("m/86h/0h/0h/0").unwrap();
-    let internal_path = DerivationPath::from_str("m/86h/0h/0h/1").unwrap();
+    let external_path = DerivationPath::from_str("m/86h/1h/0h/0").unwrap();
+    let internal_path = DerivationPath::from_str("m/86h/1h/0h/1").unwrap();
 
     // generate external and internal descriptor from mnemonic
     let (external_descriptor, ext_keymap) =

crates/wallet/examples/policy.rs

@@ -9,16 +9,14 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-extern crate bdk;
-extern crate env_logger;
-extern crate log;
+extern crate bdk_wallet;
 use std::error::Error;
 
-use bdk::bitcoin::Network;
-use bdk::descriptor::{policy::BuildSatisfaction, ExtractPolicy, IntoWalletDescriptor};
-use bdk::wallet::signer::SignersContainer;
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::descriptor::{policy::BuildSatisfaction, ExtractPolicy, IntoWalletDescriptor};
+use bdk_wallet::wallet::signer::SignersContainer;
 
-/// This example describes the use of the BDK's [`bdk::descriptor::policy`] module.
+/// This example describes the use of the BDK's [`bdk_wallet::descriptor::policy`] module.
 ///
 /// Policy is higher abstraction representation of the wallet descriptor spending condition.
 /// This is useful to express complex miniscript spending conditions into more human readable form.
@@ -29,10 +27,6 @@ use bdk::wallet::signer::SignersContainer;
 /// one of the Extend Private key.
 
 fn main() -> Result<(), Box<dyn Error>> {
-    env_logger::init_from_env(
-        env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
-    );
-
     let secp = bitcoin::secp256k1::Secp256k1::new();
 
     // The descriptor used in the example
@@ -40,15 +34,15 @@ fn main() -> Result<(), Box<dyn Error>> {
     let desc = "wsh(multi(2,tprv8ZgxMBicQKsPdpkqS7Eair4YxjcuuvDPNYmKX3sCniCf16tHEVrjjiSXEkFRnUH77yXc6ZcwHHcLNfjdi5qUvw3VDfgYiH5mNsj5izuiu2N/1/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/1/*))";
 
     // Use the descriptor string to derive the full descriptor and a keymap.
-    // The wallet descriptor can be used to create a new bdk::wallet.
+    // The wallet descriptor can be used to create a new bdk_wallet::wallet.
     // While the `keymap` can be used to create a `SignerContainer`.
     //
     // The `SignerContainer` can sign for `PSBT`s.
-    // a bdk::wallet internally uses these to handle transaction signing.
+    // a bdk_wallet::wallet internally uses these to handle transaction signing.
     // But they can be used as independent tools also.
     let (wallet_desc, keymap) = desc.into_wallet_descriptor(&secp, Network::Testnet)?;
 
-    log::info!("Example Descriptor for policy analysis : {}", wallet_desc);
+    println!("Example Descriptor for policy analysis : {}", wallet_desc);
 
     // Create the signer with the keymap and descriptor.
     let signers_container = SignersContainer::build(keymap, &wallet_desc, &secp);
@@ -60,7 +54,7 @@ fn main() -> Result<(), Box<dyn Error>> {
         .extract_policy(&signers_container, BuildSatisfaction::None, &secp)?
         .expect("We expect a policy");
 
-    log::info!("Derived Policy for the descriptor {:#?}", policy);
+    println!("Derived Policy for the descriptor {:#?}", policy);
 
     Ok(())
 }

crates/wallet/src/descriptor/checksum.rs

@@ -42,22 +42,16 @@ fn poly_mod(mut c: u64, val: u64) -> u64 {
     c
 }
 
-/// Computes the checksum bytes of a descriptor.
-/// `exclude_hash = true` ignores all data after the first '#' (inclusive).
-pub(crate) fn calc_checksum_bytes_internal(
-    mut desc: &str,
-    exclude_hash: bool,
-) -> Result<[u8; 8], DescriptorError> {
+/// Compute the checksum bytes of a descriptor, excludes any existing checksum in the descriptor string from the calculation
+pub fn calc_checksum_bytes(mut desc: &str) -> Result<[u8; 8], DescriptorError> {
     let mut c = 1;
     let mut cls = 0;
     let mut clscount = 0;
 
     let mut original_checksum = None;
-    if exclude_hash {
-        if let Some(split) = desc.split_once('#') {
-            desc = split.0;
-            original_checksum = Some(split.1);
-        }
+    if let Some(split) = desc.split_once('#') {
+        desc = split.0;
+        original_checksum = Some(split.1);
     }
 
     for ch in desc.as_bytes() {
@@ -95,39 +89,10 @@ pub(crate) fn calc_checksum_bytes_internal(
     Ok(checksum)
 }
 
-/// Compute the checksum bytes of a descriptor, excludes any existing checksum in the descriptor string from the calculation
-pub fn calc_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
-    calc_checksum_bytes_internal(desc, true)
-}
-
 /// Compute the checksum of a descriptor, excludes any existing checksum in the descriptor string from the calculation
 pub fn calc_checksum(desc: &str) -> Result<String, DescriptorError> {
     // unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
-    calc_checksum_bytes_internal(desc, true)
-        .map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
-}
-
-// TODO in release 0.25.0, remove get_checksum_bytes and get_checksum
-// TODO in release 0.25.0, consolidate calc_checksum_bytes_internal into calc_checksum_bytes
-
-/// Compute the checksum bytes of a descriptor
-#[deprecated(
-    since = "0.24.0",
-    note = "Use new `calc_checksum_bytes` function which excludes any existing checksum in the descriptor string before calculating the checksum hash bytes. See https://github.com/bitcoindevkit/bdk/pull/765."
-)]
-pub fn get_checksum_bytes(desc: &str) -> Result<[u8; 8], DescriptorError> {
-    calc_checksum_bytes_internal(desc, false)
-}
-
-/// Compute the checksum of a descriptor
-#[deprecated(
-    since = "0.24.0",
-    note = "Use new `calc_checksum` function which excludes any existing checksum in the descriptor string before calculating the checksum hash. See https://github.com/bitcoindevkit/bdk/pull/765."
-)]
-pub fn get_checksum(desc: &str) -> Result<String, DescriptorError> {
-    // unsafe is okay here as the checksum only uses bytes in `CHECKSUM_CHARSET`
-    calc_checksum_bytes_internal(desc, false)
-        .map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
+    calc_checksum_bytes(desc).map(|b| unsafe { String::from_utf8_unchecked(b.to_vec()) })
 }
 
 #[cfg(test)]

crates/wallet/src/descriptor/dsl.rs

@@ -274,14 +274,13 @@ macro_rules! impl_sortedmulti {
 #[macro_export]
 macro_rules! parse_tap_tree {
     ( @merge $tree_a:expr, $tree_b:expr) => {{
-        use $crate::alloc::sync::Arc;
         use $crate::miniscript::descriptor::TapTree;
 
         $tree_a
             .and_then(|tree_a| Ok((tree_a, $tree_b?)))
             .and_then(|((a_tree, mut a_keymap, a_networks), (b_tree, b_keymap, b_networks))| {
                 a_keymap.extend(b_keymap.into_iter());
-                Ok((TapTree::Tree(Arc::new(a_tree), Arc::new(b_tree)), a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
+                Ok((TapTree::combine(a_tree, b_tree), a_keymap, $crate::keys::merge_networks(&a_networks, &b_networks)))
             })
 
     }};
@@ -424,7 +423,7 @@ macro_rules! apply_modifier {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// let (my_descriptor, my_keys_map, networks) = bdk::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
+/// let (my_descriptor, my_keys_map, networks) = bdk_wallet::descriptor!(sh(wsh(and_v(v:pk("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy"),older(50)))))?;
 /// # Ok::<(), Box<dyn std::error::Error>>(())
 /// ```
 ///
@@ -445,7 +444,7 @@ macro_rules! apply_modifier {
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 /// let my_timelock = 50;
 ///
-/// let (descriptor_a, key_map_a, networks) = bdk::descriptor! {
+/// let (descriptor_a, key_map_a, networks) = bdk_wallet::descriptor! {
 ///     wsh (
 ///         thresh(2, pk(my_key_1), s:pk(my_key_2), s:n:d:v:older(my_timelock))
 ///     )
@@ -453,11 +452,12 @@ macro_rules! apply_modifier {
 ///
 /// #[rustfmt::skip]
 /// let b_items = vec![
-///     bdk::fragment!(pk(my_key_1))?,
-///     bdk::fragment!(s:pk(my_key_2))?,
-///     bdk::fragment!(s:n:d:v:older(my_timelock))?,
+///     bdk_wallet::fragment!(pk(my_key_1))?,
+///     bdk_wallet::fragment!(s:pk(my_key_2))?,
+///     bdk_wallet::fragment!(s:n:d:v:older(my_timelock))?,
 /// ];
-/// let (descriptor_b, mut key_map_b, networks) = bdk::descriptor!(wsh(thresh_vec(2, b_items)))?;
+/// let (descriptor_b, mut key_map_b, networks) =
+///     bdk_wallet::descriptor!(wsh(thresh_vec(2, b_items)))?;
 ///
 /// assert_eq!(descriptor_a, descriptor_b);
 /// assert_eq!(key_map_a.len(), key_map_b.len());
@@ -476,7 +476,7 @@ macro_rules! apply_modifier {
 /// let my_key_2 =
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 ///
-/// let (descriptor, key_map, networks) = bdk::descriptor! {
+/// let (descriptor, key_map, networks) = bdk_wallet::descriptor! {
 ///     wsh (
 ///         multi(2, my_key_1, my_key_2)
 ///     )
@@ -492,7 +492,7 @@ macro_rules! apply_modifier {
 /// let my_key =
 ///     bitcoin::PrivateKey::from_wif("cVt4o7BGAig1UXywgGSmARhxMdzP5qvQsxKkSsc1XEkw3tDTQFpy")?;
 ///
-/// let (descriptor, key_map, networks) = bdk::descriptor!(wpkh(my_key))?;
+/// let (descriptor, key_map, networks) = bdk_wallet::descriptor!(wpkh(my_key))?;
 /// # Ok::<(), Box<dyn std::error::Error>>(())
 /// ```
 ///
@@ -516,13 +516,14 @@ macro_rules! descriptor {
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey};
 
         $crate::impl_top_level_pk!(Pkh, $crate::miniscript::Legacy, $key)
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .map(|(a, b, c)| (Descriptor::<DescriptorPublicKey>::Pkh(a), b, c))
     });
     ( wpkh ( $key:expr ) ) => ({
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey};
 
         $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
-            .and_then(|(a, b, c)| Ok((a?, b, c)))
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .map(|(a, b, c)| (Descriptor::<DescriptorPublicKey>::Wpkh(a), b, c))
     });
     ( sh ( wpkh ( $key:expr ) ) ) => ({
@@ -532,7 +533,7 @@ macro_rules! descriptor {
         use $crate::miniscript::descriptor::{Descriptor, DescriptorPublicKey, Sh};
 
         $crate::impl_top_level_pk!(Wpkh, $crate::miniscript::Segwitv0, $key)
-            .and_then(|(a, b, c)| Ok((a?, b, c)))
+            .and_then(|(a, b, c)| Ok((a.map_err(|e| miniscript::Error::from(e))?, b, c)))
             .and_then(|(a, b, c)| Ok((Descriptor::<DescriptorPublicKey>::Sh(Sh::new_wpkh(a.into_inner())?), b, c)))
     });
     ( sh ( $( $minisc:tt )* ) ) => ({
@@ -702,10 +703,10 @@ macro_rules! fragment {
         $crate::keys::make_pkh($key, &secp)
     });
     ( after ( $value:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(After, $crate::bitcoin::PackedLockTime($value)) // TODO!! https://github.com/rust-bitcoin/rust-bitcoin/issues/1302
+        $crate::impl_leaf_opcode_value!(After, $crate::miniscript::AbsLockTime::from_consensus($value).expect("valid `AbsLockTime`"))
     });
     ( older ( $value:expr ) ) => ({
-        $crate::impl_leaf_opcode_value!(Older, $crate::bitcoin::Sequence($value)) // TODO!!
+        $crate::impl_leaf_opcode_value!(Older, $crate::miniscript::RelLockTime::from_consensus($value).expect("valid `RelLockTime`")) // TODO!!
     });
     ( sha256 ( $hash:expr ) ) => ({
         $crate::impl_leaf_opcode_value!(Sha256, $hash)
@@ -756,7 +757,8 @@ macro_rules! fragment {
             (keys_acc, net_acc)
         });
 
-        $crate::impl_leaf_opcode_value_two!(Thresh, $thresh, items)
+        let thresh = $crate::miniscript::Threshold::new($thresh, items).expect("valid threshold and pks collection");
+        $crate::impl_leaf_opcode_value!(Thresh, thresh)
             .map(|(minisc, _, _)| (minisc, key_maps, valid_networks))
     });
     ( thresh ( $thresh:expr, $( $inner:tt )* ) ) => ({
@@ -768,7 +770,12 @@ macro_rules! fragment {
     ( multi_vec ( $thresh:expr, $keys:expr ) ) => ({
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        $crate::keys::make_multi($thresh, $crate::miniscript::Terminal::Multi, $keys, &secp)
+        let fun = |k, pks| {
+            let thresh = $crate::miniscript::Threshold::new(k, pks).expect("valid threshold and pks collection");
+            $crate::miniscript::Terminal::Multi(thresh)
+        };
+
+        $crate::keys::make_multi($thresh, fun, $keys, &secp)
     });
     ( multi ( $thresh:expr $(, $key:expr )+ ) ) => ({
         $crate::group_multi_keys!( $( $key ),* )
@@ -777,7 +784,12 @@ macro_rules! fragment {
     ( multi_a_vec ( $thresh:expr, $keys:expr ) ) => ({
         let secp = $crate::bitcoin::secp256k1::Secp256k1::new();
 
-        $crate::keys::make_multi($thresh, $crate::miniscript::Terminal::MultiA, $keys, &secp)
+        let fun = |k, pks| {
+            let thresh = $crate::miniscript::Threshold::new(k, pks).expect("valid threshold and pks collection");
+            $crate::miniscript::Terminal::MultiA(thresh)
+        };
+
+        $crate::keys::make_multi($thresh, fun, $keys, &secp)
     });
     ( multi_a ( $thresh:expr $(, $key:expr )+ ) ) => ({
         $crate::group_multi_keys!( $( $key ),* )
@@ -796,7 +808,6 @@ macro_rules! fragment {
 #[cfg(test)]
 mod test {
     use alloc::string::ToString;
-    use bitcoin::hashes::hex::ToHex;
     use bitcoin::secp256k1::Secp256k1;
     use miniscript::descriptor::{DescriptorPublicKey, KeyMap};
     use miniscript::{Descriptor, Legacy, Segwitv0};
@@ -805,8 +816,8 @@ mod test {
 
     use crate::descriptor::{DescriptorError, DescriptorMeta};
     use crate::keys::{DescriptorKey, IntoDescriptorKey, ValidNetworks};
-    use bitcoin::network::constants::Network::{Bitcoin, Regtest, Signet, Testnet};
-    use bitcoin::util::bip32;
+    use bitcoin::bip32;
+    use bitcoin::Network::{Bitcoin, Regtest, Signet, Testnet};
     use bitcoin::PrivateKey;
 
     // test the descriptor!() macro
@@ -822,18 +833,15 @@ mod test {
         assert_eq!(desc.is_witness(), is_witness);
         assert_eq!(!desc.has_wildcard(), is_fixed);
         for i in 0..expected.len() {
-            let index = i as u32;
-            let child_desc = if !desc.has_wildcard() {
-                desc.at_derivation_index(0)
-            } else {
-                desc.at_derivation_index(index)
-            };
+            let child_desc = desc
+                .at_derivation_index(i as u32)
+                .expect("i is not hardened");
             let address = child_desc.address(Regtest);
             if let Ok(address) = address {
                 assert_eq!(address.to_string(), *expected.get(i).unwrap());
             } else {
                 let script = child_desc.script_pubkey();
-                assert_eq!(script.to_hex().as_str(), *expected.get(i).unwrap());
+                assert_eq!(script.to_hex_string(), *expected.get(i).unwrap());
             }
         }
     }
@@ -939,7 +947,7 @@ mod test {
 
     #[test]
     fn test_bip32_legacy_descriptors() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
 
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
@@ -984,7 +992,7 @@ mod test {
 
     #[test]
     fn test_bip32_segwitv0_descriptors() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
 
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path.clone()).into_descriptor_key().unwrap();
@@ -1041,10 +1049,10 @@ mod test {
 
     #[test]
     fn test_dsl_sortedmulti() {
-        let key_1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let key_1 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path_1 = bip32::DerivationPath::from_str("m/0").unwrap();
 
-        let key_2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let key_2 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
         let path_2 = bip32::DerivationPath::from_str("m/1").unwrap();
 
         let desc_key1 = (key_1, path_1);
@@ -1100,7 +1108,7 @@ mod test {
     // - verify the valid_networks returned is correctly computed based on the keys present in the descriptor
     #[test]
     fn test_valid_networks() {
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
@@ -1110,7 +1118,7 @@ mod test {
             [Testnet, Regtest, Signet].iter().cloned().collect()
         );
 
-        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
+        let xprv = bip32::Xpriv::from_str("xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi").unwrap();
         let path = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
         let desc_key = (xprv, path).into_descriptor_key().unwrap();
 
@@ -1123,15 +1131,15 @@ mod test {
     fn test_key_maps_merged() {
         let secp = Secp256k1::new();
 
-        let xprv1 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv1 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path1 = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key1 = (xprv1, path1.clone()).into_descriptor_key().unwrap();
 
-        let xprv2 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
+        let xprv2 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPegBHHnq7YEgM815dG24M2Jk5RVqipgDxF1HJ1tsnT815X5Fd5FRfMVUs8NZs9XCb6y9an8hRPThnhfwfXJ36intaekySHGF").unwrap();
         let path2 = bip32::DerivationPath::from_str("m/2147483647'/0").unwrap();
         let desc_key2 = (xprv2, path2.clone()).into_descriptor_key().unwrap();
 
-        let xprv3 = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
+        let xprv3 = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPdZXrcHNLf5JAJWFAoJ2TrstMRdSKtEggz6PddbuSkvHKM9oKJyFgZV1B7rw8oChspxyYbtmEXYyg1AjfWbL3ho3XHDpHRZf").unwrap();
         let path3 = bip32::DerivationPath::from_str("m/10/20/30/40").unwrap();
         let desc_key3 = (xprv3, path3.clone()).into_descriptor_key().unwrap();
 
@@ -1155,7 +1163,7 @@ mod test {
     #[test]
     fn test_script_context_validation() {
         // this compiles
-        let xprv = bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let xprv = bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
         let desc_key: DescriptorKey<Legacy> = (xprv, path).into_descriptor_key().unwrap();
 
@@ -1178,9 +1186,7 @@ mod test {
     }
 
     #[test]
-    #[should_panic(
-        expected = "Miniscript(ContextError(CompressedOnly(\"04b4632d08485ff1df2db55b9dafd23347d1c47a457072a1e87be26896549a87378ec38ff91d43e8c2092ebda601780485263da089465619e0358a5c1be7ac91f4\")))"
-    )]
+    #[should_panic(expected = "Miniscript(ContextError(UncompressedKeysNotAllowed))")]
     fn test_dsl_miniscript_checks() {
         let mut uncompressed_pk =
             PrivateKey::from_wif("L5EZftvrYaSudiozVRzTqLcHLNDoVn7H5HSfM9BAN6tMJX8oTWz6").unwrap();

crates/wallet/src/descriptor/error.rs

@@ -10,7 +10,6 @@
 // licenses.
 
 //! Descriptor errors
-
 use core::fmt;
 
 /// Errors related to the parsing and usage of descriptors
@@ -22,7 +21,8 @@ pub enum Error {
     InvalidDescriptorChecksum,
     /// The descriptor contains hardened derivation steps on public extended keys
     HardenedDerivationXpub,
-
+    /// The descriptor contains multipath keys
+    MultiPath,
     /// Error thrown while working with [`keys`](crate::keys)
     Key(crate::keys::KeyError),
     /// Error while extracting and manipulating policies
@@ -32,15 +32,17 @@ pub enum Error {
     InvalidDescriptorCharacter(u8),
 
     /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::bip32::Error),
     /// Error during base58 decoding
-    Base58(bitcoin::util::base58::Error),
+    Base58(bitcoin::base58::Error),
     /// Key-related error
-    Pk(bitcoin::util::key::Error),
+    Pk(bitcoin::key::ParsePublicKeyError),
     /// Miniscript error
     Miniscript(miniscript::Error),
     /// Hex decoding error
-    Hex(bitcoin::hashes::hex::Error),
+    Hex(bitcoin::hex::HexToBytesError),
+    /// The provided wallet descriptors are identical
+    ExternalAndInternalAreTheSame,
 }
 
 impl From<crate::keys::KeyError> for Error {
@@ -64,6 +66,10 @@ impl fmt::Display for Error {
                 f,
                 "The descriptor contains hardened derivation steps on public extended keys"
             ),
+            Self::MultiPath => write!(
+                f,
+                "The descriptor contains multipath keys, which are not supported yet"
+            ),
             Self::Key(err) => write!(f, "Key error: {}", err),
             Self::Policy(err) => write!(f, "Policy error: {}", err),
             Self::InvalidDescriptorCharacter(char) => {
@@ -74,6 +80,9 @@ impl fmt::Display for Error {
             Self::Pk(err) => write!(f, "Key-related error: {}", err),
             Self::Miniscript(err) => write!(f, "Miniscript error: {}", err),
             Self::Hex(err) => write!(f, "Hex decoding error: {}", err),
+            Self::ExternalAndInternalAreTheSame => {
+                write!(f, "External and internal descriptors are the same")
+            }
         }
     }
 }
@@ -81,9 +90,38 @@ impl fmt::Display for Error {
 #[cfg(feature = "std")]
 impl std::error::Error for Error {}
 
-impl_error!(bitcoin::util::bip32::Error, Bip32);
-impl_error!(bitcoin::util::base58::Error, Base58);
-impl_error!(bitcoin::util::key::Error, Pk);
-impl_error!(miniscript::Error, Miniscript);
-impl_error!(bitcoin::hashes::hex::Error, Hex);
-impl_error!(crate::descriptor::policy::PolicyError, Policy);
+impl From<bitcoin::bip32::Error> for Error {
+    fn from(err: bitcoin::bip32::Error) -> Self {
+        Error::Bip32(err)
+    }
+}
+
+impl From<bitcoin::base58::Error> for Error {
+    fn from(err: bitcoin::base58::Error) -> Self {
+        Error::Base58(err)
+    }
+}
+
+impl From<bitcoin::key::ParsePublicKeyError> for Error {
+    fn from(err: bitcoin::key::ParsePublicKeyError) -> Self {
+        Error::Pk(err)
+    }
+}
+
+impl From<miniscript::Error> for Error {
+    fn from(err: miniscript::Error) -> Self {
+        Error::Miniscript(err)
+    }
+}
+
+impl From<bitcoin::hex::HexToBytesError> for Error {
+    fn from(err: bitcoin::hex::HexToBytesError) -> Self {
+        Error::Hex(err)
+    }
+}
+
+impl From<crate::descriptor::policy::PolicyError> for Error {
+    fn from(err: crate::descriptor::policy::PolicyError) -> Self {
+        Error::Policy(err)
+    }
+}

crates/wallet/src/descriptor/mod.rs

@@ -18,17 +18,17 @@ use crate::collections::BTreeMap;
 use alloc::string::String;
 use alloc::vec::Vec;
 
-use bitcoin::util::bip32::{ChildNumber, DerivationPath, ExtendedPubKey, Fingerprint, KeySource};
-use bitcoin::util::{psbt, taproot};
-use bitcoin::{secp256k1, PublicKey, XOnlyPublicKey};
+use bitcoin::bip32::{ChildNumber, DerivationPath, Fingerprint, KeySource, Xpub};
+use bitcoin::{key::XOnlyPublicKey, secp256k1, PublicKey};
+use bitcoin::{psbt, taproot};
 use bitcoin::{Network, TxOut};
 
 use miniscript::descriptor::{
-    DefiniteDescriptorKey, DescriptorSecretKey, DescriptorType, InnerXKey, SinglePubKey,
+    DefiniteDescriptorKey, DescriptorMultiXKey, DescriptorSecretKey, DescriptorType,
+    DescriptorXKey, InnerXKey, KeyMap, SinglePubKey, Wildcard,
 };
 pub use miniscript::{
-    descriptor::DescriptorXKey, descriptor::KeyMap, descriptor::Wildcard, Descriptor,
-    DescriptorPublicKey, Legacy, Miniscript, ScriptContext, Segwitv0,
+    Descriptor, DescriptorPublicKey, Legacy, Miniscript, ScriptContext, Segwitv0,
 };
 use miniscript::{ForEachKey, MiniscriptKey, TranslatePk};
 
@@ -59,16 +59,16 @@ pub type DerivedDescriptor = Descriptor<DefiniteDescriptorKey>;
 /// Alias for the type of maps that represent derivation paths in a [`psbt::Input`] or
 /// [`psbt::Output`]
 ///
-/// [`psbt::Input`]: bitcoin::util::psbt::Input
-/// [`psbt::Output`]: bitcoin::util::psbt::Output
+/// [`psbt::Input`]: bitcoin::psbt::Input
+/// [`psbt::Output`]: bitcoin::psbt::Output
 pub type HdKeyPaths = BTreeMap<secp256k1::PublicKey, KeySource>;
 
 /// Alias for the type of maps that represent taproot key origins in a [`psbt::Input`] or
 /// [`psbt::Output`]
 ///
-/// [`psbt::Input`]: bitcoin::util::psbt::Input
-/// [`psbt::Output`]: bitcoin::util::psbt::Output
-pub type TapKeyOrigins = BTreeMap<bitcoin::XOnlyPublicKey, (Vec<taproot::TapLeafHash>, KeySource)>;
+/// [`psbt::Input`]: bitcoin::psbt::Input
+/// [`psbt::Output`]: bitcoin::psbt::Output
+pub type TapKeyOrigins = BTreeMap<XOnlyPublicKey, (Vec<taproot::TapLeafHash>, KeySource)>;
 
 /// Trait for types which can be converted into an [`ExtendedDescriptor`] and a [`KeyMap`] usable by a wallet in a specific [`Network`]
 pub trait IntoWalletDescriptor {
@@ -136,14 +136,10 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             network: Network,
         }
 
-        impl<'s, 'd>
-            miniscript::Translator<DescriptorPublicKey, miniscript::DummyKey, DescriptorError>
+        impl<'s, 'd> miniscript::Translator<DescriptorPublicKey, String, DescriptorError>
             for Translator<'s, 'd>
         {
-            fn pk(
-                &mut self,
-                pk: &DescriptorPublicKey,
-            ) -> Result<miniscript::DummyKey, DescriptorError> {
+            fn pk(&mut self, pk: &DescriptorPublicKey) -> Result<String, DescriptorError> {
                 let secp = &self.secp;
 
                 let (_, _, networks) = if self.descriptor.is_taproot() {
@@ -161,7 +157,7 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
                 };
 
                 if networks.contains(&self.network) {
-                    Ok(miniscript::DummyKey)
+                    Ok(Default::default())
                 } else {
                     Err(DescriptorError::Key(KeyError::InvalidNetwork))
                 }
@@ -169,35 +165,40 @@ impl IntoWalletDescriptor for (ExtendedDescriptor, KeyMap) {
             fn sha256(
                 &mut self,
                 _sha256: &<DescriptorPublicKey as MiniscriptKey>::Sha256,
-            ) -> Result<miniscript::DummySha256Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn hash256(
                 &mut self,
                 _hash256: &<DescriptorPublicKey as MiniscriptKey>::Hash256,
-            ) -> Result<miniscript::DummyHash256Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn ripemd160(
                 &mut self,
                 _ripemd160: &<DescriptorPublicKey as MiniscriptKey>::Ripemd160,
-            ) -> Result<miniscript::DummyRipemd160Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
             fn hash160(
                 &mut self,
                 _hash160: &<DescriptorPublicKey as MiniscriptKey>::Hash160,
-            ) -> Result<miniscript::DummyHash160Hash, DescriptorError> {
+            ) -> Result<String, DescriptorError> {
                 Ok(Default::default())
             }
         }
 
         // check the network for the keys
-        self.0.translate_pk(&mut Translator {
+        use miniscript::TranslateErr;
+        match self.0.translate_pk(&mut Translator {
             secp,
             network,
             descriptor: &self.0,
-        })?;
+        }) {
+            Ok(_) => {}
+            Err(TranslateErr::TranslatorErr(e)) => return Err(e),
+            Err(TranslateErr::OuterError(e)) => return Err(e.into()),
+        }
 
         Ok(self)
     }
@@ -228,7 +229,7 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
                 let pk = match pk {
                     DescriptorPublicKey::XPub(ref xpub) => {
                         let mut xpub = xpub.clone();
-                        xpub.xkey.network = self.network;
+                        xpub.xkey.network = self.network.into();
 
                         DescriptorPublicKey::XPub(xpub)
                     }
@@ -251,18 +252,23 @@ impl IntoWalletDescriptor for DescriptorTemplateOut {
         }
 
         // fixup the network for keys that need it in the descriptor
-        let translated = desc.translate_pk(&mut Translator { network })?;
+        use miniscript::TranslateErr;
+        let translated = match desc.translate_pk(&mut Translator { network }) {
+            Ok(descriptor) => descriptor,
+            Err(TranslateErr::TranslatorErr(e)) => return Err(e),
+            Err(TranslateErr::OuterError(e)) => return Err(e.into()),
+        };
         // ...and in the key map
         let fixed_keymap = keymap
             .into_iter()
             .map(|(mut k, mut v)| {
                 match (&mut k, &mut v) {
                     (DescriptorPublicKey::XPub(xpub), DescriptorSecretKey::XPrv(xprv)) => {
-                        xpub.xkey.network = network;
-                        xprv.xkey.network = network;
+                        xpub.xkey.network = network.into();
+                        xprv.xkey.network = network.into();
                     }
                     (_, DescriptorSecretKey::Single(key)) => {
-                        key.key.network = network;
+                        key.key.network = network.into();
                     }
                     _ => {}
                 }
@@ -302,6 +308,10 @@ pub(crate) fn into_wallet_descriptor_checked<T: IntoWalletDescriptor>(
         return Err(DescriptorError::HardenedDerivationXpub);
     }
 
+    if descriptor.is_multipath() {
+        return Err(DescriptorError::MultiPath);
+    }
+
     // Run miniscript's sanity check, which will look for duplicated keys and other potential
     // issues
     descriptor.sanity_check()?;
@@ -340,6 +350,18 @@ pub(crate) trait XKeyUtils {
     fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint;
 }
 
+impl<T> XKeyUtils for DescriptorMultiXKey<T>
+where
+    T: InnerXKey,
+{
+    fn root_fingerprint(&self, secp: &SecpCtx) -> Fingerprint {
+        match self.origin {
+            Some((fingerprint, _)) => fingerprint,
+            None => self.xkey.xkey_fingerprint(secp),
+        }
+    }
+}
+
 impl<T> XKeyUtils for DescriptorXKey<T>
 where
     T: InnerXKey,
@@ -355,7 +377,7 @@ where
 pub(crate) trait DescriptorMeta {
     fn is_witness(&self) -> bool;
     fn is_taproot(&self) -> bool;
-    fn get_extended_keys(&self) -> Vec<DescriptorXKey<ExtendedPubKey>>;
+    fn get_extended_keys(&self) -> Vec<DescriptorXKey<Xpub>>;
     fn derive_from_hd_keypaths(
         &self,
         hd_keypaths: &HdKeyPaths,
@@ -396,7 +418,7 @@ impl DescriptorMeta for ExtendedDescriptor {
         self.desc_type() == DescriptorType::Tr
     }
 
-    fn get_extended_keys(&self) -> Vec<DescriptorXKey<ExtendedPubKey>> {
+    fn get_extended_keys(&self) -> Vec<DescriptorXKey<Xpub>> {
         let mut answer = Vec::new();
 
         self.for_each_key(|pk| {
@@ -416,21 +438,20 @@ impl DescriptorMeta for ExtendedDescriptor {
         secp: &SecpCtx,
     ) -> Option<DerivedDescriptor> {
         // Ensure that deriving `xpub` with `path` yields `expected`
-        let verify_key = |xpub: &DescriptorXKey<ExtendedPubKey>,
-                          path: &DerivationPath,
-                          expected: &SinglePubKey| {
-            let derived = xpub
-                .xkey
-                .derive_pub(secp, path)
-                .expect("The path should never contain hardened derivation steps")
-                .public_key;
+        let verify_key =
+            |xpub: &DescriptorXKey<Xpub>, path: &DerivationPath, expected: &SinglePubKey| {
+                let derived = xpub
+                    .xkey
+                    .derive_pub(secp, path)
+                    .expect("The path should never contain hardened derivation steps")
+                    .public_key;
 
-            match expected {
-                SinglePubKey::FullKey(pk) if &PublicKey::new(derived) == pk => true,
-                SinglePubKey::XOnly(pk) if &XOnlyPublicKey::from(derived) == pk => true,
-                _ => false,
-            }
-        };
+                match expected {
+                    SinglePubKey::FullKey(pk) if &PublicKey::new(derived) == pk => true,
+                    SinglePubKey::XOnly(pk) if &XOnlyPublicKey::from(derived) == pk => true,
+                    _ => false,
+                }
+            };
 
         let mut path_found = None;
 
@@ -466,11 +487,6 @@ impl DescriptorMeta for ExtendedDescriptor {
                         ) {
                             Some(derive_path)
                         } else {
-                            log::debug!(
-                                "Key `{}` derived with {} yields an unexpected key",
-                                root_fingerprint,
-                                derive_path
-                            );
                             None
                         }
                     });
@@ -494,7 +510,10 @@ impl DescriptorMeta for ExtendedDescriptor {
             false
         });
 
-        path_found.map(|path| self.at_derivation_index(path))
+        path_found.map(|path| {
+            self.at_derivation_index(path)
+                .expect("We ignore hardened wildcards")
+        })
     }
 
     fn derive_from_hd_keypaths(
@@ -545,7 +564,7 @@ impl DescriptorMeta for ExtendedDescriptor {
             return None;
         }
 
-        let descriptor = self.at_derivation_index(0);
+        let descriptor = self.at_derivation_index(0).expect("0 is not hardened");
         match descriptor.desc_type() {
             // TODO: add pk() here
             DescriptorType::Pkh
@@ -585,11 +604,10 @@ mod test {
     use core::str::FromStr;
 
     use assert_matches::assert_matches;
-    use bitcoin::consensus::encode::deserialize;
-    use bitcoin::hashes::hex::FromHex;
+    use bitcoin::hex::FromHex;
     use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::util::{bip32, psbt};
-    use bitcoin::Script;
+    use bitcoin::{bip32, Psbt};
+    use bitcoin::{NetworkKind, ScriptBuf};
 
     use super::*;
     use crate::psbt::PsbtUtils;
@@ -600,7 +618,7 @@ mod test {
             "wpkh(02b4632d08485ff1df2db55b9dafd23347d1c47a457072a1e87be26896549a8737)",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff010052010000000162307be8e431fbaff807cdf9cdc3fde44d7402\
                  11bc8342c31ffd6ec11fe35bcc0100000000ffffffff01328601000000000016\
@@ -623,7 +641,7 @@ mod test {
             "pkh([0f056943/44h/0h/0h]tpubDDpWvmUrPZrhSPmUzCMBHffvC3HyMAPnWDSAQNBTnj1iZeJa7BZQEttFiP4DS4GCcXQHezdXhn86Hj6LHX5EDstXPWrMaSneRWM8yUf6NFd/10/*)",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff010053010000000145843b86be54a3cd8c9e38444e1162676c00df\
                  e7964122a70df491ea12fd67090100000000ffffffff01c19598000000000017\
@@ -654,7 +672,7 @@ mod test {
             "wsh(and_v(v:pk(03b6633fef2397a0a9de9d7b6f23aef8368a6e362b0581f0f0af70d5ecfd254b14),older(6)))",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff01005302000000011c8116eea34408ab6529223c9a176606742207\
                  67a1ff1d46a6e3c4a88243ea6e01000000000600000001109698000000000017\
@@ -678,7 +696,7 @@ mod test {
             "sh(and_v(v:pk(021403881a5587297818fcaf17d239cefca22fce84a45b3b1d23e836c4af671dbb),after(630000)))",
         )
         .unwrap();
-        let psbt: psbt::PartiallySignedTransaction = deserialize(
+        let psbt = Psbt::deserialize(
             &Vec::<u8>::from_hex(
                 "70736274ff0100530100000001bc8c13df445dfadcc42afa6dc841f85d22b01d\
                  a6270ebf981740f4b7b1d800390000000000feffffff01ba9598000000000017\
@@ -708,7 +726,7 @@ mod test {
 
         let secp = Secp256k1::new();
 
-        let xprv = bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3c3gF1DUWpWNr2SG2XrG8oYPpqYh7hoWsJy9NjabErnzriJPpnGHyKz5NgdXmq1KVbqS1r4NXdCoKitWg5e86zqXHa8kxyB").unwrap();
+        let xprv = bip32::Xpriv::from_str("xprv9s21ZrQH143K3c3gF1DUWpWNr2SG2XrG8oYPpqYh7hoWsJy9NjabErnzriJPpnGHyKz5NgdXmq1KVbqS1r4NXdCoKitWg5e86zqXHa8kxyB").unwrap();
         let path = bip32::DerivationPath::from_str("m/0").unwrap();
 
         // here `to_descriptor_key` will set the valid networks for the key to only mainnet, since
@@ -725,9 +743,9 @@ mod test {
             .unwrap();
 
         let mut xprv_testnet = xprv;
-        xprv_testnet.network = Network::Testnet;
+        xprv_testnet.network = NetworkKind::Test;
 
-        let xpub_testnet = bip32::ExtendedPubKey::from_priv(&secp, &xprv_testnet);
+        let xpub_testnet = bip32::Xpub::from_priv(&secp, &xprv_testnet);
         let desc_pubkey = DescriptorPublicKey::XPub(DescriptorXKey {
             xkey: xpub_testnet,
             origin: None,
@@ -817,7 +835,7 @@ mod test {
     fn test_descriptor_from_str_from_output_of_macro() {
         let secp = Secp256k1::new();
 
-        let tpub = bip32::ExtendedPubKey::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
+        let tpub = bip32::Xpub::from_str("tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK").unwrap();
         let path = bip32::DerivationPath::from_str("m/1/2").unwrap();
         let key = (tpub, path).into_descriptor_key().unwrap();
 
@@ -845,6 +863,12 @@ mod test {
 
         assert_matches!(result, Err(DescriptorError::HardenedDerivationXpub));
 
+        let descriptor = "wpkh(tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/<0;1>/*)";
+        let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
+
+        assert_matches!(result, Err(DescriptorError::MultiPath));
+
+        // repeated pubkeys
         let descriptor = "wsh(multi(2,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*,tpubD6NzVbkrYhZ4XHndKkuB8FifXm8r5FQHwrN6oZuWCz13qb93rtgKvD4PQsqC4HP4yhV3tA2fqr2RbY5mNXfM7RxXUoeABoDtsFUq2zJq6YK/0/*))";
         let result = into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet);
 
@@ -861,16 +885,16 @@ mod test {
         let (descriptor, _) =
             into_wallet_descriptor_checked(descriptor, &secp, Network::Testnet).unwrap();
 
-        let descriptor = descriptor.at_derivation_index(0);
+        let descriptor = descriptor.at_derivation_index(0).unwrap();
 
-        let script = Script::from_str("5321022f533b667e2ea3b36e21961c9fe9dca340fbe0af5210173a83ae0337ab20a57621026bb53a98e810bd0ee61a0ed1164ba6c024786d76554e793e202dc6ce9c78c4ea2102d5b8a7d66a41ffdb6f4c53d61994022e886b4f45001fb158b95c9164d45f8ca3210324b75eead2c1f9c60e8adeb5e7009fec7a29afcdb30d829d82d09562fe8bae8521032d34f8932200833487bd294aa219dcbe000b9f9b3d824799541430009f0fa55121037468f8ea99b6c64788398b5ad25480cad08f4b0d65be54ce3a55fd206b5ae4722103f72d3d96663b0ea99b0aeb0d7f273cab11a8de37885f1dddc8d9112adb87169357ae").unwrap();
+        let script = ScriptBuf::from_hex("5321022f533b667e2ea3b36e21961c9fe9dca340fbe0af5210173a83ae0337ab20a57621026bb53a98e810bd0ee61a0ed1164ba6c024786d76554e793e202dc6ce9c78c4ea2102d5b8a7d66a41ffdb6f4c53d61994022e886b4f45001fb158b95c9164d45f8ca3210324b75eead2c1f9c60e8adeb5e7009fec7a29afcdb30d829d82d09562fe8bae8521032d34f8932200833487bd294aa219dcbe000b9f9b3d824799541430009f0fa55121037468f8ea99b6c64788398b5ad25480cad08f4b0d65be54ce3a55fd206b5ae4722103f72d3d96663b0ea99b0aeb0d7f273cab11a8de37885f1dddc8d9112adb87169357ae").unwrap();
 
         let mut psbt_input = psbt::Input::default();
         psbt_input
             .update_with_descriptor_unchecked(&descriptor)
             .unwrap();
 
-        assert_eq!(psbt_input.redeem_script, Some(script.to_v0_p2wsh()));
+        assert_eq!(psbt_input.redeem_script, Some(script.to_p2wsh()));
         assert_eq!(psbt_input.witness_script, Some(script));
     }
 }

crates/wallet/src/descriptor/policy.rs

@@ -20,10 +20,10 @@
 //!
 //! ```
 //! # use std::sync::Arc;
-//! # use bdk::descriptor::*;
-//! # use bdk::wallet::signer::*;
-//! # use bdk::bitcoin::secp256k1::Secp256k1;
-//! use bdk::descriptor::policy::BuildSatisfaction;
+//! # use bdk_wallet::descriptor::*;
+//! # use bdk_wallet::wallet::signer::*;
+//! # use bdk_wallet::bitcoin::secp256k1::Secp256k1;
+//! use bdk_wallet::descriptor::policy::BuildSatisfaction;
 //! let secp = Secp256k1::new();
 //! let desc = "wsh(and_v(v:pk(cV3oCth6zxZ1UVsHLnGothsWNsaoxRhC6aeNi5VbSdFpwUkgkEci),or_d(pk(cVMTy7uebJgvFaSBwcgvwk8qn8xSLc97dKow4MBetjrrahZoimm2),older(12960))))";
 //!
@@ -33,33 +33,32 @@
 //! let signers = Arc::new(SignersContainer::build(key_map, &extended_desc, &secp));
 //! let policy = extended_desc.extract_policy(&signers, BuildSatisfaction::None, &secp)?;
 //! println!("policy: {}", serde_json::to_string(&policy).unwrap());
-//! # Ok::<(), bdk::Error>(())
+//! # Ok::<(), anyhow::Error>(())
 //! ```
 
 use crate::collections::{BTreeMap, HashSet, VecDeque};
 use alloc::string::String;
 use alloc::vec::Vec;
 use core::cmp::max;
+use miniscript::miniscript::limits::{MAX_PUBKEYS_IN_CHECKSIGADD, MAX_PUBKEYS_PER_MULTISIG};
+
 use core::fmt;
 
 use serde::ser::SerializeMap;
 use serde::{Serialize, Serializer};
 
+use bitcoin::bip32::Fingerprint;
 use bitcoin::hashes::{hash160, ripemd160, sha256};
-use bitcoin::util::bip32::Fingerprint;
-use bitcoin::{LockTime, PublicKey, Sequence, XOnlyPublicKey};
+use bitcoin::{absolute, key::XOnlyPublicKey, relative, PublicKey, Sequence};
 
 use miniscript::descriptor::{
     DescriptorPublicKey, ShInner, SinglePub, SinglePubKey, SortedMultiVec, WshInner,
 };
-use miniscript::hash256;
+use miniscript::{hash256, Threshold};
 use miniscript::{
     Descriptor, Miniscript, Satisfier, ScriptContext, SigType, Terminal, ToPublicKey,
 };
 
-#[allow(unused_imports)]
-use log::{debug, error, info, trace};
-
 use crate::descriptor::ExtractPolicy;
 use crate::keys::ExtScriptContext;
 use crate::wallet::signer::{SignerId, SignersContainer};
@@ -68,7 +67,7 @@ use crate::wallet::utils::{After, Older, SecpCtx};
 use super::checksum::calc_checksum;
 use super::error::Error;
 use super::XKeyUtils;
-use bitcoin::util::psbt::{Input as PsbtInput, PartiallySignedTransaction as Psbt};
+use bitcoin::psbt::{self, Psbt};
 use miniscript::psbt::PsbtInputSatisfier;
 
 /// A unique identifier for a key
@@ -95,6 +94,9 @@ impl PkOrF {
                 ..
             }) => PkOrF::XOnlyPubkey(*pk),
             DescriptorPublicKey::XPub(xpub) => PkOrF::Fingerprint(xpub.root_fingerprint(secp)),
+            DescriptorPublicKey::MultiXPub(multi) => {
+                PkOrF::Fingerprint(multi.root_fingerprint(secp))
+            }
         }
     }
 }
@@ -131,12 +133,12 @@ pub enum SatisfiableItem {
     /// Absolute timeclock timestamp
     AbsoluteTimelock {
         /// The timelock value
-        value: LockTime,
+        value: absolute::LockTime,
     },
     /// Relative timelock locktime
     RelativeTimelock {
         /// The timelock value
-        value: Sequence,
+        value: relative::LockTime,
     },
     /// Multi-signature public keys with threshold count
     Multisig {
@@ -451,11 +453,14 @@ pub struct Condition {
     pub csv: Option<Sequence>,
     /// Optional timelock condition
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub timelock: Option<LockTime>,
+    pub timelock: Option<absolute::LockTime>,
 }
 
 impl Condition {
-    fn merge_nlocktime(a: LockTime, b: LockTime) -> Result<LockTime, PolicyError> {
+    fn merge_nlocktime(
+        a: absolute::LockTime,
+        b: absolute::LockTime,
+    ) -> Result<absolute::LockTime, PolicyError> {
         if !a.is_same_unit(b) {
             Err(PolicyError::MixedTimelockUnits)
         } else if a > b {
@@ -515,7 +520,7 @@ pub enum PolicyError {
 impl fmt::Display for PolicyError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            Self::NotEnoughItemsSelected(err) => write!(f, "Not enought items selected: {}", err),
+            Self::NotEnoughItemsSelected(err) => write!(f, "Not enough items selected: {}", err),
             Self::IndexOutOfRange(index) => write!(f, "Index out of range: {}", index),
             Self::AddOnLeaf => write!(f, "Add on leaf"),
             Self::AddOnPartialComplete => write!(f, "Add on partial complete"),
@@ -582,30 +587,25 @@ impl Policy {
         Ok(Some(policy))
     }
 
-    fn make_multisig<Ctx: ScriptContext + 'static>(
-        keys: &[DescriptorPublicKey],
+    fn make_multi<Ctx: ScriptContext + 'static, const MAX: usize>(
+        threshold: &Threshold<DescriptorPublicKey, MAX>,
         signers: &SignersContainer,
         build_sat: BuildSatisfaction,
-        threshold: usize,
         sorted: bool,
         secp: &SecpCtx,
     ) -> Result<Option<Policy>, PolicyError> {
-        if threshold == 0 {
-            return Ok(None);
-        }
-
-        let parsed_keys = keys.iter().map(|k| PkOrF::from_key(k, secp)).collect();
+        let parsed_keys = threshold.iter().map(|k| PkOrF::from_key(k, secp)).collect();
 
         let mut contribution = Satisfaction::Partial {
-            n: keys.len(),
-            m: threshold,
+            n: threshold.n(),
+            m: threshold.k(),
             items: vec![],
             conditions: Default::default(),
             sorted: Some(sorted),
         };
         let mut satisfaction = contribution.clone();
 
-        for (index, key) in keys.iter().enumerate() {
+        for (index, key) in threshold.iter().enumerate() {
             if signers.find(signer_id(key, secp)).is_some() {
                 contribution.add(
                     &Satisfaction::Complete {
@@ -614,7 +614,6 @@ impl Policy {
                     index,
                 )?;
             }
-
             if let Some(psbt) = build_sat.psbt() {
                 if Ctx::find_signature(psbt, key, secp) {
                     satisfaction.add(
@@ -631,12 +630,11 @@ impl Policy {
 
         let mut policy: Policy = SatisfiableItem::Multisig {
             keys: parsed_keys,
-            threshold,
+            threshold: threshold.k(),
         }
         .into();
         policy.contribution = contribution;
         policy.satisfaction = satisfaction;
-
         Ok(Some(policy))
     }
 
@@ -721,7 +719,7 @@ impl Policy {
                 timelock: Some(*value),
             }),
             SatisfiableItem::RelativeTimelock { value } => Ok(Condition {
-                csv: Some(*value),
+                csv: Some((*value).into()),
                 timelock: None,
             }),
             _ => Ok(Condition::default()),
@@ -749,6 +747,7 @@ fn signer_id(key: &DescriptorPublicKey, secp: &SecpCtx) -> SignerId {
             ..
         }) => pk.to_pubkeyhash(SigType::Ecdsa).into(),
         DescriptorPublicKey::XPub(xpub) => xpub.root_fingerprint(secp).into(),
+        DescriptorPublicKey::MultiXPub(xpub) => xpub.root_fingerprint(secp).into(),
     }
 }
 
@@ -786,9 +785,9 @@ fn make_generic_signature<M: Fn() -> SatisfiableItem, F: Fn(&Psbt) -> bool>(
 fn generic_sig_in_psbt<
     // C is for "check", it's a closure we use to *check* if a psbt input contains the signature
     // for a specific key
-    C: Fn(&PsbtInput, &SinglePubKey) -> bool,
+    C: Fn(&psbt::Input, &SinglePubKey) -> bool,
     // E is for "extract", it extracts a key from the bip32 derivations found in the psbt input
-    E: Fn(&PsbtInput, Fingerprint) -> Option<SinglePubKey>,
+    E: Fn(&psbt::Input, Fingerprint) -> Option<SinglePubKey>,
 >(
     psbt: &Psbt,
     key: &DescriptorPublicKey,
@@ -806,6 +805,13 @@ fn generic_sig_in_psbt<
                 None => false,
             }
         }
+        DescriptorPublicKey::MultiXPub(xpub) => {
+            //TODO check actual derivation matches
+            match extract(input, xpub.root_fingerprint(secp)) {
+                Some(pubkey) => check(input, &pubkey),
+                None => false,
+            }
+        }
     })
 }
 
@@ -911,12 +917,12 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
             }
             Terminal::After(value) => {
                 let mut policy: Policy = SatisfiableItem::AbsoluteTimelock {
-                    value: value.into(),
+                    value: (*value).into(),
                 }
                 .into();
                 policy.contribution = Satisfaction::Complete {
                     condition: Condition {
-                        timelock: Some(value.into()),
+                        timelock: Some((*value).into()),
                         csv: None,
                     },
                 };
@@ -928,9 +934,9 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 {
                     let after = After::new(Some(current_height), false);
                     let after_sat =
-                        Satisfier::<bitcoin::PublicKey>::check_after(&after, value.into());
+                        Satisfier::<bitcoin::PublicKey>::check_after(&after, (*value).into());
                     let inputs_sat = psbt_inputs_sat(psbt).all(|sat| {
-                        Satisfier::<bitcoin::PublicKey>::check_after(&sat, value.into())
+                        Satisfier::<bitcoin::PublicKey>::check_after(&sat, (*value).into())
                     });
                     if after_sat && inputs_sat {
                         policy.satisfaction = policy.contribution.clone();
@@ -940,11 +946,14 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 Some(policy)
             }
             Terminal::Older(value) => {
-                let mut policy: Policy = SatisfiableItem::RelativeTimelock { value: *value }.into();
+                let mut policy: Policy = SatisfiableItem::RelativeTimelock {
+                    value: (*value).into(),
+                }
+                .into();
                 policy.contribution = Satisfaction::Complete {
                     condition: Condition {
                         timelock: None,
-                        csv: Some(*value),
+                        csv: Some((*value).into()),
                     },
                 };
                 if let BuildSatisfaction::PsbtTimelocks {
@@ -954,9 +963,11 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 } = build_sat
                 {
                     let older = Older::new(Some(current_height), Some(input_max_height), false);
-                    let older_sat = Satisfier::<bitcoin::PublicKey>::check_older(&older, *value);
-                    let inputs_sat = psbt_inputs_sat(psbt)
-                        .all(|sat| Satisfier::<bitcoin::PublicKey>::check_older(&sat, *value));
+                    let older_sat =
+                        Satisfier::<bitcoin::PublicKey>::check_older(&older, (*value).into());
+                    let inputs_sat = psbt_inputs_sat(psbt).all(|sat| {
+                        Satisfier::<bitcoin::PublicKey>::check_older(&sat, (*value).into())
+                    });
                     if older_sat && inputs_sat {
                         policy.satisfaction = policy.contribution.clone();
                     }
@@ -974,9 +985,12 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
             Terminal::Hash160(hash) => {
                 Some(SatisfiableItem::Hash160Preimage { hash: *hash }.into())
             }
-            Terminal::Multi(k, pks) | Terminal::MultiA(k, pks) => {
-                Policy::make_multisig::<Ctx>(pks, signers, build_sat, *k, false, secp)?
-            }
+            Terminal::Multi(threshold) => Policy::make_multi::<Ctx, MAX_PUBKEYS_PER_MULTISIG>(
+                threshold, signers, build_sat, false, secp,
+            )?,
+            Terminal::MultiA(threshold) => Policy::make_multi::<Ctx, MAX_PUBKEYS_IN_CHECKSIGADD>(
+                threshold, signers, build_sat, false, secp,
+            )?,
             // Identities
             Terminal::Alt(inner)
             | Terminal::Swap(inner)
@@ -1004,8 +1018,9 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                 a.extract_policy(signers, build_sat, secp)?,
                 b.extract_policy(signers, build_sat, secp)?,
             )?,
-            Terminal::Thresh(k, nodes) => {
-                let mut threshold = *k;
+            Terminal::Thresh(threshold) => {
+                let mut k = threshold.k();
+                let nodes = threshold.data();
                 let mapped: Vec<_> = nodes
                     .iter()
                     .map(|n| n.extract_policy(signers, build_sat, secp))
@@ -1015,13 +1030,13 @@ impl<Ctx: ScriptContext + 'static> ExtractPolicy for Miniscript<DescriptorPublic
                     .collect();
 
                 if mapped.len() < nodes.len() {
-                    threshold = match threshold.checked_sub(nodes.len() - mapped.len()) {
+                    k = match k.checked_sub(nodes.len() - mapped.len()) {
                         None => return Ok(None),
                         Some(x) => x,
                     };
                 }
 
-                Policy::make_thresh(mapped, threshold)?
+                Policy::make_thresh(mapped, k)?
             }
 
             // Unsupported
@@ -1075,13 +1090,10 @@ impl ExtractPolicy for Descriptor<DescriptorPublicKey> {
             build_sat: BuildSatisfaction,
             secp: &SecpCtx,
         ) -> Result<Option<Policy>, Error> {
-            Ok(Policy::make_multisig::<Ctx>(
-                keys.pks.as_ref(),
-                signers,
-                build_sat,
-                keys.k,
-                true,
-                secp,
+            let threshold = Threshold::new(keys.k(), keys.pks().to_vec())
+                .expect("valid threshold and pks collection");
+            Ok(Policy::make_multi::<Ctx, MAX_PUBKEYS_PER_MULTISIG>(
+                &threshold, signers, build_sat, true, secp,
             )?)
         }
 
@@ -1125,7 +1137,7 @@ impl ExtractPolicy for Descriptor<DescriptorPublicKey> {
                 let key_spend_sig =
                     miniscript::Tap::make_signature(tr.internal_key(), signers, build_sat, secp);
 
-                if tr.taptree().is_none() {
+                if tr.tap_tree().is_none() {
                     Ok(Some(key_spend_sig))
                 } else {
                     let mut items = vec![key_spend_sig];
@@ -1156,8 +1168,8 @@ mod test {
     use crate::wallet::signer::SignersContainer;
     use alloc::{string::ToString, sync::Arc};
     use assert_matches::assert_matches;
+    use bitcoin::bip32;
     use bitcoin::secp256k1::Secp256k1;
-    use bitcoin::util::bip32;
     use bitcoin::Network;
     use core::str::FromStr;
 
@@ -1172,8 +1184,8 @@ mod test {
         secp: &SecpCtx,
     ) -> (DescriptorKey<Ctx>, DescriptorKey<Ctx>, Fingerprint) {
         let path = bip32::DerivationPath::from_str(path).unwrap();
-        let tprv = bip32::ExtendedPrivKey::from_str(tprv).unwrap();
-        let tpub = bip32::ExtendedPubKey::from_priv(secp, &tprv);
+        let tprv = bip32::Xpriv::from_str(tprv).unwrap();
+        let tpub = bip32::Xpub::from_priv(secp, &tprv);
         let fingerprint = tprv.fingerprint(secp);
         let prvkey = (tprv, path.clone()).into_descriptor_key().unwrap();
         let pubkey = (tpub, path).into_descriptor_key().unwrap();
@@ -1575,6 +1587,7 @@ mod test {
 
         let addr = wallet_desc
             .at_derivation_index(0)
+            .unwrap()
             .address(Network::Testnet)
             .unwrap();
         assert_eq!(
@@ -1641,6 +1654,7 @@ mod test {
 
         let addr = wallet_desc
             .at_derivation_index(0)
+            .unwrap()
             .address(Network::Testnet)
             .unwrap();
         assert_eq!(

crates/wallet/src/descriptor/template.rs

@@ -14,7 +14,7 @@
 //! This module contains the definition of various common script templates that are ready to be
 //! used. See the documentation of each template for an example.
 
-use bitcoin::util::bip32;
+use bitcoin::bip32;
 use bitcoin::Network;
 
 use miniscript::{Legacy, Segwitv0, Tap};
@@ -36,17 +36,17 @@ pub type DescriptorTemplateOut = (ExtendedDescriptor, KeyMap, ValidNetworks);
 /// ## Example
 ///
 /// ```
-/// use bdk::descriptor::error::Error as DescriptorError;
-/// use bdk::keys::{IntoDescriptorKey, KeyError};
-/// use bdk::miniscript::Legacy;
-/// use bdk::template::{DescriptorTemplate, DescriptorTemplateOut};
+/// use bdk_wallet::descriptor::error::Error as DescriptorError;
+/// use bdk_wallet::keys::{IntoDescriptorKey, KeyError};
+/// use bdk_wallet::miniscript::Legacy;
+/// use bdk_wallet::template::{DescriptorTemplate, DescriptorTemplateOut};
 /// use bitcoin::Network;
 ///
 /// struct MyP2PKH<K: IntoDescriptorKey<Legacy>>(K);
 ///
 /// impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for MyP2PKH<K> {
 ///     fn build(self, network: Network) -> Result<DescriptorTemplateOut, DescriptorError> {
-///         Ok(bdk::descriptor!(pkh(self.0))?)
+///         Ok(bdk_wallet::descriptor!(pkh(self.0))?)
 ///     }
 /// }
 /// ```
@@ -72,17 +72,21 @@ impl<T: DescriptorTemplate> IntoWalletDescriptor for T {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::P2Pkh;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Pkh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Pkh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::new(P2Pkh(key_external), P2Pkh(key_internal), Network::Testnet)?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "mwJ8hxFYW19JLuc65RCTaP4v1rzVU8cVMT"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
@@ -100,17 +104,25 @@ impl<K: IntoDescriptorKey<Legacy>> DescriptorTemplate for P2Pkh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// use bdk::template::P2Wpkh_P2Sh;
-/// use bdk::wallet::AddressIndex;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Wpkh_P2Sh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Wpkh_P2Sh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::new(
+///     P2Wpkh_P2Sh(key_external),
+///     P2Wpkh_P2Sh(key_internal),
+///     Network::Testnet,
+/// )?;
 ///
 /// assert_eq!(
-///     wallet.get_address(AddressIndex::New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "2NB4ox5VDRw1ecUv6SnT3VQHPXveYztRqk5"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
@@ -129,17 +141,21 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh_P2Sh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet};
-/// use bdk::template::P2Wpkh;
-/// use bdk::wallet::AddressIndex::New;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet};
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2Wpkh;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2Wpkh(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::new(P2Wpkh(key_external), P2Wpkh(key_internal), Network::Testnet)?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "tb1q4525hmgw265tl3drrl8jjta7ayffu6jf68ltjd"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
@@ -157,17 +173,21 @@ impl<K: IntoDescriptorKey<Segwitv0>> DescriptorTemplate for P2Wpkh<K> {
 /// ## Example
 ///
 /// ```
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::Wallet;
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::P2TR;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::Wallet;
+/// # use bdk_wallet::KeychainKind;
+/// use bdk_wallet::template::P2TR;
 ///
-/// let key =
+/// let key_external =
 ///     bitcoin::PrivateKey::from_wif("cTc4vURSzdx6QE6KVynWGomDbLaA75dNALMNyfjh3p8DRRar84Um")?;
-/// let mut wallet = Wallet::new_no_persist(P2TR(key), None, Network::Testnet)?;
+/// let key_internal =
+///     bitcoin::PrivateKey::from_wif("cVpPVruEDdmutPzisEsYvtST1usBR3ntr8pXSyt6D2YYqXRyPcFW")?;
+/// let mut wallet = Wallet::new(P2TR(key_external), P2TR(key_internal), Network::Testnet)?;
 ///
 /// assert_eq!(
-///     wallet.get_address(New).to_string(),
+///     wallet
+///         .next_unused_address(KeychainKind::External)
+///         .to_string(),
 ///     "tb1pvjf9t34fznr53u5tqhejz4nr69luzkhlvsdsdfq9pglutrpve2xq7hps46"
 /// );
 /// # Ok::<_, Box<dyn std::error::Error>>(())
@@ -190,20 +210,19 @@ impl<K: IntoDescriptorKey<Tap>> DescriptorTemplate for P2TR<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip44;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip44;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::new(
 ///     Bip44(key.clone(), KeychainKind::External),
-///     Some(Bip44(key, KeychainKind::Internal)),
+///     Bip44(key, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "mmogjc7HJEZkrLqyQYqJmxUqFaC7i4uf89");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "pkh([c55b303f/44'/1'/0']tpubDCuorCpzvYS2LCD75BR46KHE8GdDeg1wsAgNZeNr6DaB5gQK1o14uErKwKLuFmeemkQ6N2m3rNgvctdJLyr7nwu2yia7413Hhg8WWE44cgT/0/*)#5wrnv0xt");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "mmogjc7HJEZkrLqyQYqJmxUqFaC7i4uf89");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "pkh([c55b303f/44'/1'/0']tpubDCuorCpzvYS2LCD75BR46KHE8GdDeg1wsAgNZeNr6DaB5gQK1o14uErKwKLuFmeemkQ6N2m3rNgvctdJLyr7nwu2yia7413Hhg8WWE44cgT/0/*)#5wrnv0xt");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip44<K: DerivableKey<Legacy>>(pub K, pub KeychainKind);
@@ -227,21 +246,20 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip44Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip44Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
+/// let mut wallet = Wallet::new(
 ///     Bip44Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip44Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip44Public(key, fingerprint, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "pkh([c55b303f/44'/1'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#cfhumdqz");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "miNG7dJTzJqNbFS19svRdTCisC65dsubtR");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "pkh([c55b303f/44'/1'/0']tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU/0/*)#cfhumdqz");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip44Public<K: DerivableKey<Legacy>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
@@ -265,20 +283,19 @@ impl<K: DerivableKey<Legacy>> DescriptorTemplate for Bip44Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip49;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip49;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::new(
 ///     Bip49(key.clone(), KeychainKind::External),
-///     Some(Bip49(key, KeychainKind::Internal)),
+///     Bip49(key, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "2N4zkWAoGdUv4NXhSsU8DvS5MB36T8nKHEB");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDDYr4kdnZgjjShzYNjZUZXUUtpXaofdkMaipyS8ThEh45qFmhT4hKYways7UXmg6V7het1QiFo9kf4kYUXyDvV4rHEyvSpys9pjCB3pukxi/0/*))#s9vxlc8e");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "2N4zkWAoGdUv4NXhSsU8DvS5MB36T8nKHEB");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDDYr4kdnZgjjShzYNjZUZXUUtpXaofdkMaipyS8ThEh45qFmhT4hKYways7UXmg6V7het1QiFo9kf4kYUXyDvV4rHEyvSpys9pjCB3pukxi/0/*))#s9vxlc8e");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip49<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
@@ -302,21 +319,20 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip49Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip49Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
+/// let mut wallet = Wallet::new(
 ///     Bip49Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip49Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip49Public(key, fingerprint, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#3tka9g0q");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "2N3K4xbVAHoiTQSwxkZjWDfKoNC27pLkYnt");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "sh(wpkh([c55b303f/49'/1'/0']tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L/0/*))#3tka9g0q");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip49Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
@@ -340,20 +356,19 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip49Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip84;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip84;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::new(
 ///     Bip84(key.clone(), KeychainKind::External),
-///     Some(Bip84(key, KeychainKind::Internal)),
+///     Bip84(key, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1qhl85z42h7r4su5u37rvvw0gk8j2t3n9y7zsg4n");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "wpkh([c55b303f/84'/1'/0']tpubDDc5mum24DekpNw92t6fHGp8Gr2JjF9J7i4TZBtN6Vp8xpAULG5CFaKsfugWa5imhrQQUZKXe261asP5koDHo5bs3qNTmf3U3o4v9SaB8gg/0/*)#6kfecsmr");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1qhl85z42h7r4su5u37rvvw0gk8j2t3n9y7zsg4n");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "wpkh([c55b303f/84'/1'/0']tpubDDc5mum24DekpNw92t6fHGp8Gr2JjF9J7i4TZBtN6Vp8xpAULG5CFaKsfugWa5imhrQQUZKXe261asP5koDHo5bs3qNTmf3U3o4v9SaB8gg/0/*)#6kfecsmr");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip84<K: DerivableKey<Segwitv0>>(pub K, pub KeychainKind);
@@ -377,21 +392,20 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip84Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip84Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
+/// let mut wallet = Wallet::new(
 ///     Bip84Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip84Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip84Public(key, fingerprint, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "wpkh([c55b303f/84'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#dhu402yv");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1qedg9fdlf8cnnqfd5mks6uz5w4kgpk2pr6y4qc7");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "wpkh([c55b303f/84'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#dhu402yv");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip84Public<K: DerivableKey<Segwitv0>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
@@ -415,20 +429,19 @@ impl<K: DerivableKey<Segwitv0>> DescriptorTemplate for Bip84Public<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip86;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip86;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPeZRHk4rTG6orPS2CRNFX3njhUXx5vj9qGog5ZMH4uGReDWN5kCkY3jmWEtWause41CDvBRXD1shKknAMKxT99o9qUTRVC6m")?;
+/// let mut wallet = Wallet::new(
 ///     Bip86(key.clone(), KeychainKind::External),
-///     Some(Bip86(key, KeychainKind::Internal)),
+///     Bip86(key, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1p5unlj09djx8xsjwe97269kqtxqpwpu2epeskgqjfk4lnf69v4tnqpp35qu");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "tr([c55b303f/86'/1'/0']tpubDCiHofpEs47kx358bPdJmTZHmCDqQ8qw32upCSxHrSEdeeBs2T5Mq6QMB2ukeMqhNBiyhosBvJErteVhfURPGXPv3qLJPw5MVpHUewsbP2m/0/*)#dkgvr5hm");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1p5unlj09djx8xsjwe97269kqtxqpwpu2epeskgqjfk4lnf69v4tnqpp35qu");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "tr([c55b303f/86'/1'/0']tpubDCiHofpEs47kx358bPdJmTZHmCDqQ8qw32upCSxHrSEdeeBs2T5Mq6QMB2ukeMqhNBiyhosBvJErteVhfURPGXPv3qLJPw5MVpHUewsbP2m/0/*)#dkgvr5hm");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip86<K: DerivableKey<Tap>>(pub K, pub KeychainKind);
@@ -452,21 +465,20 @@ impl<K: DerivableKey<Tap>> DescriptorTemplate for Bip86<K> {
 ///
 /// ```
 /// # use std::str::FromStr;
-/// # use bdk::bitcoin::{PrivateKey, Network};
-/// # use bdk::{Wallet,  KeychainKind};
-/// # use bdk::wallet::AddressIndex::New;
-/// use bdk::template::Bip86Public;
+/// # use bdk_wallet::bitcoin::{PrivateKey, Network};
+/// # use bdk_wallet::{Wallet,  KeychainKind};
+/// use bdk_wallet::template::Bip86Public;
 ///
-/// let key = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
-/// let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f")?;
-/// let mut wallet = Wallet::new_no_persist(
+/// let key = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q")?;
+/// let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f")?;
+/// let mut wallet = Wallet::new(
 ///     Bip86Public(key.clone(), fingerprint, KeychainKind::External),
-///     Some(Bip86Public(key, fingerprint, KeychainKind::Internal)),
+///     Bip86Public(key, fingerprint, KeychainKind::Internal),
 ///     Network::Testnet,
 /// )?;
 ///
-/// assert_eq!(wallet.get_address(New).to_string(), "tb1pwjp9f2k5n0xq73ecuu0c5njvgqr3vkh7yaylmpqvsuuaafymh0msvcmh37");
-/// assert_eq!(wallet.public_descriptor(KeychainKind::External).unwrap().to_string(), "tr([c55b303f/86'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#2p65srku");
+/// assert_eq!(wallet.next_unused_address(KeychainKind::External).to_string(), "tb1pwjp9f2k5n0xq73ecuu0c5njvgqr3vkh7yaylmpqvsuuaafymh0msvcmh37");
+/// assert_eq!(wallet.public_descriptor(KeychainKind::External).to_string(), "tr([c55b303f/86'/1'/0']tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q/0/*)#2p65srku");
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
 pub struct Bip86Public<K: DerivableKey<Tap>>(pub K, pub bip32::Fingerprint, pub KeychainKind);
@@ -565,31 +577,31 @@ mod test {
     // BIP44 `pkh(key/44'/{0,1}'/0'/{0,1}/*)`
     #[test]
     fn test_bip44_template_cointype() {
-        use bitcoin::util::bip32::ChildNumber::{self, Hardened};
+        use bitcoin::bip32::ChildNumber::{self, Hardened};
 
-        let xprvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
-        assert_eq!(Network::Bitcoin, xprvkey.network);
+        let xprvkey = bitcoin::bip32::Xpriv::from_str("xprv9s21ZrQH143K2fpbqApQL69a4oKdGVnVN52R82Ft7d1pSqgKmajF62acJo3aMszZb6qQ22QsVECSFxvf9uyxFUvFYQMq3QbtwtRSMjLAhMf").unwrap();
+        assert!(xprvkey.network.is_mainnet());
         let xdesc = Bip44(xprvkey, KeychainKind::Internal)
             .build(Network::Bitcoin)
             .unwrap();
 
         if let ExtendedDescriptor::Pkh(pkh) = xdesc.0 {
-            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().into();
-            let purpose = path.get(0).unwrap();
+            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 0 });
         }
 
-        let tprvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
-        assert_eq!(Network::Testnet, tprvkey.network);
+        let tprvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        assert!(!tprvkey.network.is_mainnet());
         let tdesc = Bip44(tprvkey, KeychainKind::Internal)
             .build(Network::Testnet)
             .unwrap();
 
         if let ExtendedDescriptor::Pkh(pkh) = tdesc.0 {
-            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().into();
-            let purpose = path.get(0).unwrap();
+            let path: Vec<ChildNumber> = pkh.into_inner().full_derivation_path().unwrap().into();
+            let purpose = path.first().unwrap();
             assert_matches!(purpose, Hardened { index: 44 });
             let coin_type = path.get(1).unwrap();
             assert_matches!(coin_type, Hardened { index: 1 });
@@ -612,9 +624,9 @@ mod test {
         for i in 0..expected.len() {
             let index = i as u32;
             let child_desc = if !desc.has_wildcard() {
-                desc.at_derivation_index(0)
+                desc.at_derivation_index(0).unwrap()
             } else {
-                desc.at_derivation_index(index)
+                desc.at_derivation_index(index).unwrap()
             };
             let address = child_desc.address(network).unwrap();
             assert_eq!(address.to_string(), *expected.get(i).unwrap());
@@ -740,7 +752,7 @@ mod test {
     // BIP44 `pkh(key/44'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip44_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip44(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -770,8 +782,8 @@ mod test {
     // BIP44 public `pkh(key/{0,1}/*)`
     #[test]
     fn test_bip44_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDDDzQ31JkZB7VxUr9bjvBivDdqoFLrDPyLWtLapArAi51ftfmCb2DPxwLQzX65iNcXz1DGaVvyvo6JQ6rTU73r2gqdEo8uov9QKRb7nKCSU").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip44Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -801,7 +813,7 @@ mod test {
     // BIP49 `sh(wpkh(key/49'/0'/0'/{0,1}/*))`
     #[test]
     fn test_bip49_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip49(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -831,8 +843,8 @@ mod test {
     // BIP49 public `sh(wpkh(key/{0,1}/*))`
     #[test]
     fn test_bip49_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDC49r947KGK52X5rBWS4BLs5m9SRY3pYHnvRrm7HcybZ3BfdEsGFyzCMzayi1u58eT82ZeyFZwH7DD6Q83E3fM9CpfMtmnTygnLfP59jL9L").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip49Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -862,7 +874,7 @@ mod test {
     // BIP84 `wpkh(key/84'/0'/0'/{0,1}/*)`
     #[test]
     fn test_bip84_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("tprv8ZgxMBicQKsPcx5nBGsR63Pe8KnRUqmbJNENAfGftF3yuXoMMoVJJcYeUw5eVkm9WBPjWYt6HMWYJNesB5HaNVBaFc1M6dRjWSYnmewUMYy").unwrap();
         check(
             Bip84(prvkey, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -892,8 +904,8 @@ mod test {
     // BIP84 public `wpkh(key/{0,1}/*)`
     #[test]
     fn test_bip84_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("c55b303f").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("tpubDC2Qwo2TFsaNC4ju8nrUJ9mqVT3eSgdmy1yPqhgkjwmke3PRXutNGRYAUo6RCHTcVQaDR3ohNU9we59brGHuEKPvH1ags2nevW5opEE9Z5Q").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("c55b303f").unwrap();
         check(
             Bip84Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             true,
@@ -924,7 +936,7 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_template() {
-        let prvkey = bitcoin::util::bip32::ExtendedPrivKey::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
+        let prvkey = bitcoin::bip32::Xpriv::from_str("xprv9s21ZrQH143K3GJpoapnV8SFfukcVBSfeCficPSGfubmSFDxo1kuHnLisriDvSnRRuL2Qrg5ggqHKNVpxR86QEC8w35uxmGoggxtQTPvfUu").unwrap();
         check(
             Bip86(prvkey, KeychainKind::External).build(Network::Bitcoin),
             false,
@@ -955,8 +967,8 @@ mod test {
     // Used addresses in test vector in https://github.com/bitcoin/bips/blob/master/bip-0086.mediawiki
     #[test]
     fn test_bip86_public_template() {
-        let pubkey = bitcoin::util::bip32::ExtendedPubKey::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
-        let fingerprint = bitcoin::util::bip32::Fingerprint::from_str("73c5da0a").unwrap();
+        let pubkey = bitcoin::bip32::Xpub::from_str("xpub6BgBgsespWvERF3LHQu6CnqdvfEvtMcQjYrcRzx53QJjSxarj2afYWcLteoGVky7D3UKDP9QyrLprQ3VCECoY49yfdDEHGCtMMj92pReUsQ").unwrap();
+        let fingerprint = bitcoin::bip32::Fingerprint::from_str("73c5da0a").unwrap();
         check(
             Bip86Public(pubkey, fingerprint, KeychainKind::External).build(Network::Bitcoin),
             false,

crates/wallet/src/keys/bip39.rs

@@ -15,7 +15,7 @@
 // something that should be fairly simple to re-implement.
 
 use alloc::string::String;
-use bitcoin::util::bip32;
+use bitcoin::bip32;
 use bitcoin::Network;
 
 use miniscript::ScriptContext;
@@ -57,7 +57,7 @@ pub type MnemonicWithPassphrase = (Mnemonic, Option<String>);
 #[cfg_attr(docsrs, doc(cfg(feature = "keys-bip39")))]
 impl<Ctx: ScriptContext> DerivableKey<Ctx> for Seed {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-        Ok(bip32::ExtendedPrivKey::new_master(Network::Bitcoin, &self[..])?.into())
+        Ok(bip32::Xpriv::new_master(Network::Bitcoin, &self[..])?.into())
     }
 
     fn into_descriptor_key(
@@ -142,7 +142,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for Mnemonic {
         (word_count, language): Self::Options,
         entropy: Self::Entropy,
     ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
-        let entropy = &entropy.as_ref()[..(word_count as usize / 8)];
+        let entropy = &entropy[..(word_count as usize / 8)];
         let mnemonic = Mnemonic::from_entropy_in(language, entropy)?;
 
         Ok(GeneratedKey::new(mnemonic, any_network()))
@@ -154,7 +154,7 @@ mod test {
     use alloc::string::ToString;
     use core::str::FromStr;
 
-    use bitcoin::util::bip32;
+    use bitcoin::bip32;
 
     use bip39::{Language, Mnemonic};
 

crates/wallet/src/keys/mod.rs

@@ -22,8 +22,8 @@ use core::str::FromStr;
 
 use bitcoin::secp256k1::{self, Secp256k1, Signing};
 
-use bitcoin::util::bip32;
-use bitcoin::{Network, PrivateKey, PublicKey, XOnlyPublicKey};
+use bitcoin::bip32;
+use bitcoin::{key::XOnlyPublicKey, Network, PrivateKey, PublicKey};
 
 use miniscript::descriptor::{Descriptor, DescriptorXKey, Wildcard};
 pub use miniscript::descriptor::{
@@ -97,7 +97,7 @@ impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
         }
     }
 
-    // This method is used internally by `bdk::fragment!` and `bdk::descriptor!`. It has to be
+    // This method is used internally by `bdk_wallet::fragment!` and `bdk_wallet::descriptor!`. It has to be
     // public because it is effectively called by external crates once the macros are expanded,
     // but since it is not meant to be part of the public api we hide it from the docs.
     #[doc(hidden)]
@@ -110,7 +110,7 @@ impl<Ctx: ScriptContext> DescriptorKey<Ctx> {
                 Ok((public, KeyMap::default(), valid_networks))
             }
             DescriptorKey::Secret(secret, valid_networks, _) => {
-                let mut key_map = KeyMap::with_capacity(1);
+                let mut key_map = KeyMap::new();
 
                 let public = secret
                     .to_public(secp)
@@ -206,9 +206,9 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type valid in any context:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
+/// use bdk_wallet::keys::{DescriptorKey, IntoDescriptorKey, KeyError, ScriptContext};
 ///
 /// pub struct MyKeyType {
 ///     pubkey: PublicKey,
@@ -224,9 +224,9 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type that is only valid on mainnet:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{
+/// use bdk_wallet::keys::{
 ///     mainnet_network, DescriptorKey, DescriptorPublicKey, IntoDescriptorKey, KeyError,
 ///     ScriptContext, SinglePub, SinglePubKey,
 /// };
@@ -251,9 +251,11 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// Key type that internally encodes in which context it's valid. The context is checked at runtime:
 ///
 /// ```
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 ///
-/// use bdk::keys::{DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext};
+/// use bdk_wallet::keys::{
+///     DescriptorKey, ExtScriptContext, IntoDescriptorKey, KeyError, ScriptContext,
+/// };
 ///
 /// pub struct MyKeyType {
 ///     is_legacy: bool,
@@ -279,17 +281,17 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// makes the compiler (correctly) fail.
 ///
 /// ```compile_fail
-/// use bdk::bitcoin::PublicKey;
+/// use bdk_wallet::bitcoin::PublicKey;
 /// use core::str::FromStr;
 ///
-/// use bdk::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
+/// use bdk_wallet::keys::{DescriptorKey, IntoDescriptorKey, KeyError};
 ///
 /// pub struct MySegwitOnlyKeyType {
 ///     pubkey: PublicKey,
 /// }
 ///
-/// impl IntoDescriptorKey<bdk::miniscript::Segwitv0> for MySegwitOnlyKeyType {
-///     fn into_descriptor_key(self) -> Result<DescriptorKey<bdk::miniscript::Segwitv0>, KeyError> {
+/// impl IntoDescriptorKey<bdk_wallet::miniscript::Segwitv0> for MySegwitOnlyKeyType {
+///     fn into_descriptor_key(self) -> Result<DescriptorKey<bdk_wallet::miniscript::Segwitv0>, KeyError> {
 ///         self.pubkey.into_descriptor_key()
 ///     }
 /// }
@@ -297,8 +299,8 @@ impl<Ctx: ScriptContext + 'static> ExtScriptContext for Ctx {
 /// let key = MySegwitOnlyKeyType {
 ///     pubkey: PublicKey::from_str("...")?,
 /// };
-/// let (descriptor, _, _) = bdk::descriptor!(pkh(key))?;
-/// //                                       ^^^^^ changing this to `wpkh` would make it compile
+/// let (descriptor, _, _) = bdk_wallet::descriptor!(pkh(key))?;
+/// //                                               ^^^^^ changing this to `wpkh` would make it compile
 ///
 /// # Ok::<_, Box<dyn std::error::Error>>(())
 /// ```
@@ -309,15 +311,15 @@ pub trait IntoDescriptorKey<Ctx: ScriptContext>: Sized {
 
 /// Enum for extended keys that can be either `xprv` or `xpub`
 ///
-/// An instance of [`ExtendedKey`] can be constructed from an [`ExtendedPrivKey`](bip32::ExtendedPrivKey)
-/// or an [`ExtendedPubKey`](bip32::ExtendedPubKey) by using the `From` trait.
+/// An instance of [`ExtendedKey`] can be constructed from an [`Xpriv`](bip32::Xpriv)
+/// or an [`Xpub`](bip32::Xpub) by using the `From` trait.
 ///
 /// Defaults to the [`Legacy`](miniscript::Legacy) context.
 pub enum ExtendedKey<Ctx: ScriptContext = miniscript::Legacy> {
     /// A private extended key, aka an `xprv`
-    Private((bip32::ExtendedPrivKey, PhantomData<Ctx>)),
+    Private((bip32::Xpriv, PhantomData<Ctx>)),
     /// A public extended key, aka an `xpub`
-    Public((bip32::ExtendedPubKey, PhantomData<Ctx>)),
+    Public((bip32::Xpub, PhantomData<Ctx>)),
 }
 
 impl<Ctx: ScriptContext> ExtendedKey<Ctx> {
@@ -329,43 +331,43 @@ impl<Ctx: ScriptContext> ExtendedKey<Ctx> {
         }
     }
 
-    /// Transform the [`ExtendedKey`] into an [`ExtendedPrivKey`](bip32::ExtendedPrivKey) for the
+    /// Transform the [`ExtendedKey`] into an [`Xpriv`](bip32::Xpriv) for the
     /// given [`Network`], if the key contains the private data
-    pub fn into_xprv(self, network: Network) -> Option<bip32::ExtendedPrivKey> {
+    pub fn into_xprv(self, network: Network) -> Option<bip32::Xpriv> {
         match self {
             ExtendedKey::Private((mut xprv, _)) => {
-                xprv.network = network;
+                xprv.network = network.into();
                 Some(xprv)
             }
             ExtendedKey::Public(_) => None,
         }
     }
 
-    /// Transform the [`ExtendedKey`] into an [`ExtendedPubKey`](bip32::ExtendedPubKey) for the
+    /// Transform the [`ExtendedKey`] into an [`Xpub`](bip32::Xpub) for the
     /// given [`Network`]
     pub fn into_xpub<C: Signing>(
         self,
         network: bitcoin::Network,
         secp: &Secp256k1<C>,
-    ) -> bip32::ExtendedPubKey {
+    ) -> bip32::Xpub {
         let mut xpub = match self {
-            ExtendedKey::Private((xprv, _)) => bip32::ExtendedPubKey::from_priv(secp, &xprv),
+            ExtendedKey::Private((xprv, _)) => bip32::Xpub::from_priv(secp, &xprv),
             ExtendedKey::Public((xpub, _)) => xpub,
         };
 
-        xpub.network = network;
+        xpub.network = network.into();
         xpub
     }
 }
 
-impl<Ctx: ScriptContext> From<bip32::ExtendedPubKey> for ExtendedKey<Ctx> {
-    fn from(xpub: bip32::ExtendedPubKey) -> Self {
+impl<Ctx: ScriptContext> From<bip32::Xpub> for ExtendedKey<Ctx> {
+    fn from(xpub: bip32::Xpub) -> Self {
         ExtendedKey::Public((xpub, PhantomData))
     }
 }
 
-impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
-    fn from(xprv: bip32::ExtendedPrivKey) -> Self {
+impl<Ctx: ScriptContext> From<bip32::Xpriv> for ExtendedKey<Ctx> {
+    fn from(xprv: bip32::Xpriv) -> Self {
         ExtendedKey::Private((xprv, PhantomData))
     }
 }
@@ -383,28 +385,28 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 ///
 /// ## Examples
 ///
-/// Key types that can be directly converted into an [`ExtendedPrivKey`] or
-/// an [`ExtendedPubKey`] can implement only the required `into_extended_key()` method.
+/// Key types that can be directly converted into an [`Xpriv`] or
+/// an [`Xpub`] can implement only the required `into_extended_key()` method.
 ///
 /// ```
-/// use bdk::bitcoin;
-/// use bdk::bitcoin::util::bip32;
-/// use bdk::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
+/// use bdk_wallet::bitcoin;
+/// use bdk_wallet::bitcoin::bip32;
+/// use bdk_wallet::keys::{DerivableKey, ExtendedKey, KeyError, ScriptContext};
 ///
 /// struct MyCustomKeyType {
 ///     key_data: bitcoin::PrivateKey,
-///     chain_code: Vec<u8>,
+///     chain_code: [u8; 32],
 ///     network: bitcoin::Network,
 /// }
 ///
 /// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
 ///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-///         let xprv = bip32::ExtendedPrivKey {
-///             network: self.network,
+///         let xprv = bip32::Xpriv {
+///             network: self.network.into(),
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
-///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             chain_code: bip32::ChainCode::from(&self.chain_code),
 ///             child_number: bip32::ChildNumber::Normal { index: 0 },
 ///         };
 ///
@@ -413,30 +415,30 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 /// }
 /// ```
 ///
-/// Types that don't internally encode the [`Network`](bitcoin::Network) in which they are valid need some extra
+/// Types that don't internally encode the [`Network`] in which they are valid need some extra
 /// steps to override the set of valid networks, otherwise only the network specified in the
-/// [`ExtendedPrivKey`] or [`ExtendedPubKey`] will be considered valid.
+/// [`Xpriv`] or [`Xpub`] will be considered valid.
 ///
 /// ```
-/// use bdk::bitcoin;
-/// use bdk::bitcoin::util::bip32;
-/// use bdk::keys::{
+/// use bdk_wallet::bitcoin;
+/// use bdk_wallet::bitcoin::bip32;
+/// use bdk_wallet::keys::{
 ///     any_network, DerivableKey, DescriptorKey, ExtendedKey, KeyError, ScriptContext,
 /// };
 ///
 /// struct MyCustomKeyType {
 ///     key_data: bitcoin::PrivateKey,
-///     chain_code: Vec<u8>,
+///     chain_code: [u8; 32],
 /// }
 ///
 /// impl<Ctx: ScriptContext> DerivableKey<Ctx> for MyCustomKeyType {
 ///     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
-///         let xprv = bip32::ExtendedPrivKey {
-///             network: bitcoin::Network::Bitcoin, // pick an arbitrary network here
+///         let xprv = bip32::Xpriv {
+///             network: bitcoin::Network::Bitcoin.into(), // pick an arbitrary network here
 ///             depth: 0,
 ///             parent_fingerprint: bip32::Fingerprint::default(),
 ///             private_key: self.key_data.inner,
-///             chain_code: bip32::ChainCode::from(self.chain_code.as_ref()),
+///             chain_code: bip32::ChainCode::from(&self.chain_code),
 ///             child_number: bip32::ChildNumber::Normal { index: 0 },
 ///         };
 ///
@@ -459,8 +461,8 @@ impl<Ctx: ScriptContext> From<bip32::ExtendedPrivKey> for ExtendedKey<Ctx> {
 /// ```
 ///
 /// [`DerivationPath`]: (bip32::DerivationPath)
-/// [`ExtendedPrivKey`]: (bip32::ExtendedPrivKey)
-/// [`ExtendedPubKey`]: (bip32::ExtendedPubKey)
+/// [`Xpriv`]: (bip32::Xpriv)
+/// [`Xpub`]: (bip32::Xpub)
 pub trait DerivableKey<Ctx: ScriptContext = miniscript::Legacy>: Sized {
     /// Consume `self` and turn it into an [`ExtendedKey`]
     #[cfg_attr(
@@ -469,9 +471,9 @@ pub trait DerivableKey<Ctx: ScriptContext = miniscript::Legacy>: Sized {
 This can be used to get direct access to `xprv`s and `xpub`s for types that implement this trait,
 like [`Mnemonic`](bip39::Mnemonic) when the `keys-bip39` feature is enabled.
 ```rust
-use bdk::bitcoin::Network;
-use bdk::keys::{DerivableKey, ExtendedKey};
-use bdk::keys::bip39::{Mnemonic, Language};
+use bdk_wallet::bitcoin::Network;
+use bdk_wallet::keys::{DerivableKey, ExtendedKey};
+use bdk_wallet::keys::bip39::{Mnemonic, Language};
 
 # fn main() -> Result<(), Box<dyn std::error::Error>> {
 let xkey: ExtendedKey =
@@ -520,13 +522,13 @@ impl<Ctx: ScriptContext> DerivableKey<Ctx> for ExtendedKey<Ctx> {
     }
 }
 
-impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPubKey {
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::Xpub {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
         Ok(self.into())
     }
 }
 
-impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::ExtendedPrivKey {
+impl<Ctx: ScriptContext> DerivableKey<Ctx> for bip32::Xpriv {
     fn into_extended_key(self) -> Result<ExtendedKey<Ctx>, KeyError> {
         Ok(self.into())
     }
@@ -670,7 +672,7 @@ where
 {
 }
 
-impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
+impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::Xpriv {
     type Entropy = [u8; 32];
 
     type Options = ();
@@ -681,7 +683,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for bip32::ExtendedPrivKey {
         entropy: Self::Entropy,
     ) -> Result<GeneratedKey<Self, Ctx>, Self::Error> {
         // pick a arbitrary network here, but say that we support all of them
-        let xprv = bip32::ExtendedPrivKey::new_master(Network::Bitcoin, entropy.as_ref())?;
+        let xprv = bip32::Xpriv::new_master(Network::Bitcoin, entropy.as_ref())?;
         Ok(GeneratedKey::new(xprv, any_network()))
     }
 }
@@ -715,7 +717,7 @@ impl<Ctx: ScriptContext> GeneratableKey<Ctx> for PrivateKey {
         let inner = secp256k1::SecretKey::from_slice(&entropy)?;
         let private_key = PrivateKey {
             compressed: options.compressed,
-            network: Network::Bitcoin,
+            network: Network::Bitcoin.into(),
             inner,
         };
 
@@ -754,7 +756,7 @@ fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     let (key_map, valid_networks) = key_maps_networks.into_iter().fold(
         (KeyMap::default(), any_network()),
         |(mut keys_acc, net_acc), (key, net)| {
-            keys_acc.extend(key.into_iter());
+            keys_acc.extend(key);
             let net_acc = merge_networks(&net_acc, &net);
 
             (keys_acc, net_acc)
@@ -764,7 +766,7 @@ fn expand_multi_keys<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((pks, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `pk_k()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `pk_k()` fragments
 #[doc(hidden)]
 pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     descriptor_key: Pk,
@@ -778,7 +780,7 @@ pub fn make_pk<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `pk_h()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `pk_h()` fragments
 #[doc(hidden)]
 pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     descriptor_key: Pk,
@@ -792,7 +794,7 @@ pub fn make_pkh<Pk: IntoDescriptorKey<Ctx>, Ctx: ScriptContext>(
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::fragment!` to build `multi()` fragments
+// Used internally by `bdk_wallet::fragment!` to build `multi()` fragments
 #[doc(hidden)]
 pub fn make_multi<
     Pk: IntoDescriptorKey<Ctx>,
@@ -812,7 +814,7 @@ pub fn make_multi<
     Ok((minisc, key_map, valid_networks))
 }
 
-// Used internally by `bdk::descriptor!` to build `sortedmulti()` fragments
+// Used internally by `bdk_wallet::descriptor!` to build `sortedmulti()` fragments
 #[doc(hidden)]
 pub fn make_sortedmulti<Pk, Ctx, F>(
     thresh: usize,
@@ -834,7 +836,7 @@ where
     Ok((descriptor, key_map, valid_networks))
 }
 
-/// The "identity" conversion is used internally by some `bdk::fragment`s
+/// The "identity" conversion is used internally by some `bdk_wallet::fragment`s
 impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorKey<Ctx> {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         Ok(self)
@@ -845,9 +847,7 @@ impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorPublicKey {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         let networks = match self {
             DescriptorPublicKey::Single(_) => any_network(),
-            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. })
-                if xkey.network == Network::Bitcoin =>
-            {
+            DescriptorPublicKey::XPub(DescriptorXKey { xkey, .. }) if xkey.network.is_mainnet() => {
                 mainnet_network()
             }
             _ => test_networks(),
@@ -880,12 +880,8 @@ impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for XOnlyPublicKey {
 impl<Ctx: ScriptContext> IntoDescriptorKey<Ctx> for DescriptorSecretKey {
     fn into_descriptor_key(self) -> Result<DescriptorKey<Ctx>, KeyError> {
         let networks = match &self {
-            DescriptorSecretKey::Single(sk) if sk.key.network == Network::Bitcoin => {
-                mainnet_network()
-            }
-            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. })
-                if xkey.network == Network::Bitcoin =>
-            {
+            DescriptorSecretKey::Single(sk) if sk.key.network.is_mainnet() => mainnet_network(),
+            DescriptorSecretKey::XPrv(DescriptorXKey { xkey, .. }) if xkey.network.is_mainnet() => {
                 mainnet_network()
             }
             _ => test_networks(),
@@ -927,13 +923,22 @@ pub enum KeyError {
     Message(String),
 
     /// BIP32 error
-    Bip32(bitcoin::util::bip32::Error),
+    Bip32(bitcoin::bip32::Error),
     /// Miniscript error
     Miniscript(miniscript::Error),
 }
 
-impl_error!(miniscript::Error, Miniscript, KeyError);
-impl_error!(bitcoin::util::bip32::Error, Bip32, KeyError);
+impl From<miniscript::Error> for KeyError {
+    fn from(err: miniscript::Error) -> Self {
+        KeyError::Miniscript(err)
+    }
+}
+
+impl From<bip32::Error> for KeyError {
+    fn from(err: bip32::Error) -> Self {
+        KeyError::Bip32(err)
+    }
+}
 
 impl fmt::Display for KeyError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
@@ -953,7 +958,7 @@ impl std::error::Error for KeyError {}
 
 #[cfg(test)]
 pub mod test {
-    use bitcoin::util::bip32;
+    use bitcoin::bip32;
 
     use super::*;
 
@@ -962,7 +967,7 @@ pub mod test {
     #[test]
     fn test_keys_generate_xprv() {
         let generated_xprv: GeneratedKey<_, miniscript::Segwitv0> =
-            bip32::ExtendedPrivKey::generate_with_entropy_default(TEST_ENTROPY).unwrap();
+            bip32::Xpriv::generate_with_entropy_default(TEST_ENTROPY).unwrap();
 
         assert_eq!(generated_xprv.valid_networks, any_network());
         assert_eq!(generated_xprv.to_string(), "xprv9s21ZrQH143K4Xr1cJyqTvuL2FWR8eicgY9boWqMBv8MDVUZ65AXHnzBrK1nyomu6wdcabRgmGTaAKawvhAno1V5FowGpTLVx3jxzE5uk3Q");
@@ -992,6 +997,6 @@ pub mod test {
         .unwrap();
         let xprv = xkey.into_xprv(Network::Testnet).unwrap();
 
-        assert_eq!(xprv.network, Network::Testnet);
+        assert_eq!(xprv.network, Network::Testnet.into());
     }
 }

crates/wallet/src/lib.rs

@@ -17,9 +17,6 @@ extern crate std;
 pub extern crate alloc;
 
 pub extern crate bitcoin;
-#[cfg(feature = "hardware-signer")]
-pub extern crate hwi;
-extern crate log;
 pub extern crate miniscript;
 extern crate serde;
 extern crate serde_json;
@@ -27,9 +24,6 @@ extern crate serde_json;
 #[cfg(feature = "keys-bip39")]
 extern crate bip39;
 
-#[allow(unused_imports)]
-#[macro_use]
-pub(crate) mod error;
 pub mod descriptor;
 pub mod keys;
 pub mod psbt;
@@ -38,7 +32,6 @@ pub mod wallet;
 
 pub use descriptor::template;
 pub use descriptor::HdKeyPaths;
-pub use error::Error;
 pub use types::*;
 pub use wallet::signer;
 pub use wallet::signer::SignOptions;

crates/wallet/src/psbt/mod.rs

@@ -9,11 +9,12 @@
 // You may not use this file except in accordance with one or both of these
 // licenses.
 
-//! Additional functions on the `rust-bitcoin` `PartiallySignedTransaction` structure.
+//! Additional functions on the `rust-bitcoin` `Psbt` structure.
 
-use crate::FeeRate;
 use alloc::vec::Vec;
-use bitcoin::util::psbt::PartiallySignedTransaction as Psbt;
+use bitcoin::Amount;
+use bitcoin::FeeRate;
+use bitcoin::Psbt;
 use bitcoin::TxOut;
 
 // TODO upstream the functions here to `rust-bitcoin`?
@@ -25,44 +26,36 @@ pub trait PsbtUtils {
 
     /// The total transaction fee amount, sum of input amounts minus sum of output amounts, in sats.
     /// If the PSBT is missing a TxOut for an input returns None.
-    fn fee_amount(&self) -> Option<u64>;
+    fn fee_amount(&self) -> Option<Amount>;
 
     /// The transaction's fee rate. This value will only be accurate if calculated AFTER the
-    /// `PartiallySignedTransaction` is finalized and all witness/signature data is added to the
+    /// `Psbt` is finalized and all witness/signature data is added to the
     /// transaction.
     /// If the PSBT is missing a TxOut for an input returns None.
     fn fee_rate(&self) -> Option<FeeRate>;
 }
 
 impl PsbtUtils for Psbt {
-    #[allow(clippy::all)] // We want to allow `manual_map` but it is too new.
     fn get_utxo_for(&self, input_index: usize) -> Option<TxOut> {
         let tx = &self.unsigned_tx;
+        let input = self.inputs.get(input_index)?;
 
-        if input_index >= tx.input.len() {
-            return None;
-        }
-
-        if let Some(input) = self.inputs.get(input_index) {
-            if let Some(wit_utxo) = &input.witness_utxo {
-                Some(wit_utxo.clone())
-            } else if let Some(in_tx) = &input.non_witness_utxo {
-                Some(in_tx.output[tx.input[input_index].previous_output.vout as usize].clone())
-            } else {
-                None
-            }
-        } else {
-            None
+        match (&input.witness_utxo, &input.non_witness_utxo) {
+            (Some(_), _) => input.witness_utxo.clone(),
+            (_, Some(_)) => input.non_witness_utxo.as_ref().map(|in_tx| {
+                in_tx.output[tx.input[input_index].previous_output.vout as usize].clone()
+            }),
+            _ => None,
         }
     }
 
-    fn fee_amount(&self) -> Option<u64> {
+    fn fee_amount(&self) -> Option<Amount> {
         let tx = &self.unsigned_tx;
         let utxos: Option<Vec<TxOut>> = (0..tx.input.len()).map(|i| self.get_utxo_for(i)).collect();
 
         utxos.map(|inputs| {
-            let input_amount: u64 = inputs.iter().map(|i| i.value).sum();
-            let output_amount: u64 = self.unsigned_tx.output.iter().map(|o| o.value).sum();
+            let input_amount: Amount = inputs.iter().map(|i| i.value).sum();
+            let output_amount: Amount = self.unsigned_tx.output.iter().map(|o| o.value).sum();
             input_amount
                 .checked_sub(output_amount)
                 .expect("input amount must be greater than output amount")
@@ -71,9 +64,7 @@ impl PsbtUtils for Psbt {
 
     fn fee_rate(&self) -> Option<FeeRate> {
         let fee_amount = self.fee_amount();
-        fee_amount.map(|fee| {
-            let weight = self.clone().extract_tx().weight();
-            FeeRate::from_wu(fee, weight)
-        })
+        let weight = self.clone().extract_tx().ok()?.weight();
+        fee_amount.map(|fee| fee / weight)
     }
 }

crates/wallet/src/types.rs

@@ -0,0 +1,135 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+use alloc::boxed::Box;
+use core::convert::AsRef;
+
+use bdk_chain::ConfirmationTime;
+use bitcoin::blockdata::transaction::{OutPoint, Sequence, TxOut};
+use bitcoin::psbt;
+
+use serde::{Deserialize, Serialize};
+
+/// Types of keychains
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+pub enum KeychainKind {
+    /// External keychain, used for deriving recipient addresses.
+    External = 0,
+    /// Internal keychain, used for deriving change addresses.
+    Internal = 1,
+}
+
+impl KeychainKind {
+    /// Return [`KeychainKind`] as a byte
+    pub fn as_byte(&self) -> u8 {
+        match self {
+            KeychainKind::External => b'e',
+            KeychainKind::Internal => b'i',
+        }
+    }
+}
+
+impl AsRef<[u8]> for KeychainKind {
+    fn as_ref(&self) -> &[u8] {
+        match self {
+            KeychainKind::External => b"e",
+            KeychainKind::Internal => b"i",
+        }
+    }
+}
+
+/// An unspent output owned by a [`Wallet`].
+///
+/// [`Wallet`]: crate::Wallet
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash)]
+pub struct LocalOutput {
+    /// Reference to a transaction output
+    pub outpoint: OutPoint,
+    /// Transaction output
+    pub txout: TxOut,
+    /// Type of keychain
+    pub keychain: KeychainKind,
+    /// Whether this UTXO is spent or not
+    pub is_spent: bool,
+    /// The derivation index for the script pubkey in the wallet
+    pub derivation_index: u32,
+    /// The confirmation time for transaction containing this utxo
+    pub confirmation_time: ConfirmationTime,
+}
+
+/// A [`Utxo`] with its `satisfaction_weight`.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct WeightedUtxo {
+    /// The weight of the witness data and `scriptSig` expressed in [weight units]. This is used to
+    /// properly maintain the feerate when adding this input to a transaction during coin selection.
+    ///
+    /// [weight units]: https://en.bitcoin.it/wiki/Weight_units
+    pub satisfaction_weight: usize,
+    /// The UTXO
+    pub utxo: Utxo,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+/// An unspent transaction output (UTXO).
+pub enum Utxo {
+    /// A UTXO owned by the local wallet.
+    Local(LocalOutput),
+    /// A UTXO owned by another wallet.
+    Foreign {
+        /// The location of the output.
+        outpoint: OutPoint,
+        /// The nSequence value to set for this input.
+        sequence: Option<Sequence>,
+        /// The information about the input we require to add it to a PSBT.
+        // Box it to stop the type being too big.
+        psbt_input: Box<psbt::Input>,
+    },
+}
+
+impl Utxo {
+    /// Get the location of the UTXO
+    pub fn outpoint(&self) -> OutPoint {
+        match &self {
+            Utxo::Local(local) => local.outpoint,
+            Utxo::Foreign { outpoint, .. } => *outpoint,
+        }
+    }
+
+    /// Get the `TxOut` of the UTXO
+    pub fn txout(&self) -> &TxOut {
+        match &self {
+            Utxo::Local(local) => &local.txout,
+            Utxo::Foreign {
+                outpoint,
+                psbt_input,
+                ..
+            } => {
+                if let Some(prev_tx) = &psbt_input.non_witness_utxo {
+                    return &prev_tx.output[outpoint.vout as usize];
+                }
+
+                if let Some(txout) = &psbt_input.witness_utxo {
+                    return txout;
+                }
+
+                unreachable!("Foreign UTXOs will always have one of these set")
+            }
+        }
+    }
+
+    /// Get the sequence number if an explicit sequence number has to be set for this input.
+    pub fn sequence(&self) -> Option<Sequence> {
+        match self {
+            Utxo::Local(_) => None,
+            Utxo::Foreign { sequence, .. } => *sequence,
+        }
+    }
+}

crates/wallet/src/wallet/coin_selection.rs

@@ -26,10 +26,11 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::{self, coin_selection::*};
-//! # use bdk::*;
-//! # use bdk::wallet::coin_selection::decide_change;
-//! # const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
+//! # use bdk_wallet::wallet::{self, ChangeSet, coin_selection::*, coin_selection};
+//! # use bdk_wallet::wallet::error::CreateTxError;
+//! # use bdk_wallet::*;
+//! # use bdk_wallet::wallet::coin_selection::decide_change;
+//! # use anyhow::Error;
 //! #[derive(Debug)]
 //! struct AlwaysSpendEverything;
 //!
@@ -41,25 +42,29 @@
 //!         fee_rate: FeeRate,
 //!         target_amount: u64,
 //!         drain_script: &Script,
-//!     ) -> Result<CoinSelectionResult, bdk::Error> {
+//!     ) -> Result<CoinSelectionResult, coin_selection::Error> {
 //!         let mut selected_amount = 0;
-//!         let mut additional_weight = 0;
+//!         let mut additional_weight = Weight::ZERO;
 //!         let all_utxos_selected = required_utxos
 //!             .into_iter()
 //!             .chain(optional_utxos)
 //!             .scan(
 //!                 (&mut selected_amount, &mut additional_weight),
 //!                 |(selected_amount, additional_weight), weighted_utxo| {
-//!                     **selected_amount += weighted_utxo.utxo.txout().value;
-//!                     **additional_weight += TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight;
+//!                     **selected_amount += weighted_utxo.utxo.txout().value.to_sat();
+//!                     **additional_weight += Weight::from_wu(
+//!                         (TxIn::default().segwit_weight().to_wu()
+//!                             + weighted_utxo.satisfaction_weight as u64)
+//!                             as u64,
+//!                     );
 //!                     Some(weighted_utxo.utxo)
 //!                 },
 //!             )
 //!             .collect::<Vec<_>>();
-//!         let additional_fees = fee_rate.fee_wu(additional_weight);
+//!         let additional_fees = (fee_rate * additional_weight).to_sat();
 //!         let amount_needed_with_fees = additional_fees + target_amount;
 //!         if selected_amount < amount_needed_with_fees {
-//!             return Err(bdk::Error::InsufficientFunds {
+//!             return Err(coin_selection::Error::InsufficientFunds {
 //!                 needed: amount_needed_with_fees,
 //!                 available: selected_amount,
 //!             });
@@ -80,37 +85,77 @@
 //! # let mut wallet = doctest_wallet!();
 //! // create wallet, sync, ...
 //!
-//! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt").unwrap();
-//! let (psbt, details) = {
+//! let to_address = Address::from_str("2N4eQYCbKUHCCTUjBJeHcJp9ok6J2GZsTDt")
+//!     .unwrap()
+//!     .require_network(Network::Testnet)
+//!     .unwrap();
+//! let psbt = {
 //!     let mut builder = wallet.build_tx().coin_selection(AlwaysSpendEverything);
-//!     builder.add_recipient(to_address.script_pubkey(), 50_000);
+//!     builder.add_recipient(to_address.script_pubkey(), Amount::from_sat(50_000));
 //!     builder.finish()?
 //! };
 //!
 //! // inspect, sign, broadcast, ...
 //!
-//! # Ok::<(), bdk::Error>(())
+//! # Ok::<(), anyhow::Error>(())
 //! ```
 
-use crate::types::FeeRate;
+use crate::chain::collections::HashSet;
 use crate::wallet::utils::IsDust;
+use crate::Utxo;
 use crate::WeightedUtxo;
-use crate::{error::Error, Utxo};
+use bitcoin::FeeRate;
 
 use alloc::vec::Vec;
 use bitcoin::consensus::encode::serialize;
-use bitcoin::Script;
+use bitcoin::OutPoint;
+use bitcoin::TxIn;
+use bitcoin::{Script, Weight};
 
 use core::convert::TryInto;
+use core::fmt::{self, Formatter};
 use rand::seq::SliceRandom;
 
 /// Default coin selection algorithm used by [`TxBuilder`](super::tx_builder::TxBuilder) if not
 /// overridden
 pub type DefaultCoinSelectionAlgorithm = BranchAndBoundCoinSelection;
 
-// Base weight of a Txin, not counting the weight needed for satisfying it.
-// prev_txid (32 bytes) + prev_vout (4 bytes) + sequence (4 bytes)
-pub(crate) const TXIN_BASE_WEIGHT: usize = (32 + 4 + 4) * 4;
+/// Errors that can be thrown by the [`coin_selection`](crate::wallet::coin_selection) module
+#[derive(Debug)]
+pub enum Error {
+    /// Wallet's UTXO set is not enough to cover recipient's requested plus fee
+    InsufficientFunds {
+        /// Sats needed for some transaction
+        needed: u64,
+        /// Sats available for spending
+        available: u64,
+    },
+    /// Branch and bound coin selection tries to avoid needing a change by finding the right inputs for
+    /// the desired outputs plus fee, if there is not such combination this error is thrown
+    BnBNoExactMatch,
+    /// Branch and bound coin selection possible attempts with sufficiently big UTXO set could grow
+    /// exponentially, thus a limit is set, and when hit, this error is thrown
+    BnBTotalTriesExceeded,
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::InsufficientFunds { needed, available } => write!(
+                f,
+                "Insufficient funds: {} sat available of {} sat needed",
+                available, needed
+            ),
+            Self::BnBTotalTriesExceeded => {
+                write!(f, "Branch and bound coin selection: total tries exceeded")
+            }
+            Self::BnBNoExactMatch => write!(f, "Branch and bound coin selection: not exact match"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for Error {}
 
 #[derive(Debug)]
 /// Remaining amount after performing coin selection
@@ -147,7 +192,7 @@ pub struct CoinSelectionResult {
 impl CoinSelectionResult {
     /// The total value of the inputs selected.
     pub fn selected_amount(&self) -> u64 {
-        self.selected.iter().map(|u| u.txout().value).sum()
+        self.selected.iter().map(|u| u.txout().value.to_sat()).sum()
     }
 
     /// The total value of the inputs selected from the local wallet.
@@ -155,7 +200,7 @@ impl CoinSelectionResult {
         self.selected
             .iter()
             .filter_map(|u| match u {
-                Utxo::Local(_) => Some(u.txout().value),
+                Utxo::Local(_) => Some(u.txout().value.to_sat()),
                 _ => None,
             })
             .sum()
@@ -208,12 +253,6 @@ impl CoinSelectionAlgorithm for LargestFirstCoinSelection {
         target_amount: u64,
         drain_script: &Script,
     ) -> Result<CoinSelectionResult, Error> {
-        log::debug!(
-            "target_amount = `{}`, fee_rate = `{:?}`",
-            target_amount,
-            fee_rate
-        );
-
         // We put the "required UTXOs" first and make sure the optional UTXOs are sorted,
         // initially smallest to largest, before being reversed with `.rev()`.
         let utxos = {
@@ -271,11 +310,12 @@ impl CoinSelectionAlgorithm for OldestFirstCoinSelection {
 pub fn decide_change(remaining_amount: u64, fee_rate: FeeRate, drain_script: &Script) -> Excess {
     // drain_output_len = size(len(script_pubkey)) + len(script_pubkey) + size(output_value)
     let drain_output_len = serialize(drain_script).len() + 8usize;
-    let change_fee = fee_rate.fee_vb(drain_output_len);
+    let change_fee =
+        (fee_rate * Weight::from_vb(drain_output_len as u64).expect("overflow occurred")).to_sat();
     let drain_val = remaining_amount.saturating_sub(change_fee);
 
     if drain_val.is_dust(drain_script) {
-        let dust_threshold = drain_script.dust_value().to_sat();
+        let dust_threshold = drain_script.minimal_non_dust().to_sat();
         Excess::NoChange {
             dust_threshold,
             change_fee,
@@ -302,16 +342,13 @@ fn select_sorted_utxos(
             (&mut selected_amount, &mut fee_amount),
             |(selected_amount, fee_amount), (must_use, weighted_utxo)| {
                 if must_use || **selected_amount < target_amount + **fee_amount {
-                    **fee_amount +=
-                        fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
-                    **selected_amount += weighted_utxo.utxo.txout().value;
-
-                    log::debug!(
-                        "Selected {}, updated fee_amount = `{}`",
-                        weighted_utxo.utxo.outpoint(),
-                        fee_amount
-                    );
-
+                    **fee_amount += (fee_rate
+                        * Weight::from_wu(
+                            TxIn::default().segwit_weight().to_wu()
+                                + weighted_utxo.satisfaction_weight as u64,
+                        ))
+                    .to_sat();
+                    **selected_amount += weighted_utxo.utxo.txout().value.to_sat();
                     Some(weighted_utxo.utxo)
                 } else {
                     None
@@ -351,8 +388,12 @@ struct OutputGroup {
 
 impl OutputGroup {
     fn new(weighted_utxo: WeightedUtxo, fee_rate: FeeRate) -> Self {
-        let fee = fee_rate.fee_wu(TXIN_BASE_WEIGHT + weighted_utxo.satisfaction_weight);
-        let effective_value = weighted_utxo.utxo.txout().value as i64 - fee as i64;
+        let fee = (fee_rate
+            * Weight::from_wu(
+                TxIn::default().segwit_weight().to_wu() + weighted_utxo.satisfaction_weight as u64,
+            ))
+        .to_sat();
+        let effective_value = weighted_utxo.utxo.txout().value.to_sat() as i64 - fee as i64;
         OutputGroup {
             weighted_utxo,
             fee,
@@ -418,7 +459,8 @@ impl CoinSelectionAlgorithm for BranchAndBoundCoinSelection {
             .iter()
             .fold(0, |acc, x| acc + x.effective_value);
 
-        let cost_of_change = self.size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change =
+            (Weight::from_vb(self.size_of_change).expect("overflow occurred") * fee_rate).to_sat();
 
         // `curr_value` and `curr_available_value` are both the sum of *effective_values* of
         // the UTXOs. For the optional UTXOs (curr_available_value) we filter out UTXOs with
@@ -442,7 +484,7 @@ impl CoinSelectionAlgorithm for BranchAndBoundCoinSelection {
                     .chain(optional_utxos.iter())
                     .fold((0, 0), |(mut fees, mut value), utxo| {
                         fees += utxo.fee;
-                        value += utxo.weighted_utxo.utxo.txout().value;
+                        value += utxo.weighted_utxo.utxo.txout().value.to_sat();
 
                         (fees, value)
                     });
@@ -509,7 +551,7 @@ impl BranchAndBoundCoinSelection {
         mut curr_value: i64,
         mut curr_available_value: i64,
         target_amount: i64,
-        cost_of_change: f32,
+        cost_of_change: u64,
         drain_script: &Script,
         fee_rate: FeeRate,
     ) -> Result<CoinSelectionResult, Error> {
@@ -546,7 +588,7 @@ impl BranchAndBoundCoinSelection {
                 // If we found a solution better than the previous one, or if there wasn't previous
                 // solution, update the best solution
                 if best_selection_value.is_none() || curr_value < best_selection_value.unwrap() {
-                    best_selection = current_selection.clone();
+                    best_selection.clone_from(&current_selection);
                     best_selection_value = Some(curr_value);
                 }
 
@@ -675,25 +717,44 @@ impl BranchAndBoundCoinSelection {
     }
 }
 
+/// Remove duplicate UTXOs.
+///
+/// If a UTXO appears in both `required` and `optional`, the appearance in `required` is kept.
+pub(crate) fn filter_duplicates<I>(required: I, optional: I) -> (I, I)
+where
+    I: IntoIterator<Item = WeightedUtxo> + FromIterator<WeightedUtxo>,
+{
+    let mut visited = HashSet::<OutPoint>::new();
+    let required = required
+        .into_iter()
+        .filter(|utxo| visited.insert(utxo.utxo.outpoint()))
+        .collect::<I>();
+    let optional = optional
+        .into_iter()
+        .filter(|utxo| visited.insert(utxo.utxo.outpoint()))
+        .collect::<I>();
+    (required, optional)
+}
+
 #[cfg(test)]
 mod test {
     use assert_matches::assert_matches;
     use core::str::FromStr;
 
     use bdk_chain::ConfirmationTime;
-    use bitcoin::{OutPoint, Script, TxOut};
+    use bitcoin::{Amount, ScriptBuf, TxIn, TxOut};
 
     use super::*;
     use crate::types::*;
-    use crate::wallet::Vbytes;
+    use crate::wallet::coin_selection::filter_duplicates;
 
     use rand::rngs::StdRng;
     use rand::seq::SliceRandom;
     use rand::{Rng, RngCore, SeedableRng};
 
-    // n. of items on witness (1WU) + signature len (1WU) + signature and sighash (72WU)
-    // + pubkey len (1WU) + pubkey (33WU) + script sig len (1 byte, 4WU)
-    const P2WPKH_SATISFACTION_SIZE: usize = 1 + 1 + 72 + 1 + 33 + 4;
+    // signature len (1WU) + signature and sighash (72WU)
+    // + pubkey len (1WU) + pubkey (33WU)
+    const P2WPKH_SATISFACTION_SIZE: usize = 1 + 72 + 1 + 33;
 
     const FEE_AMOUNT: u64 = 50;
 
@@ -706,11 +767,11 @@ mod test {
         .unwrap();
         WeightedUtxo {
             satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-            utxo: Utxo::Local(LocalUtxo {
+            utxo: Utxo::Local(LocalOutput {
                 outpoint,
                 txout: TxOut {
-                    value,
-                    script_pubkey: Script::new(),
+                    value: Amount::from_sat(value),
+                    script_pubkey: ScriptBuf::new(),
                 },
                 keychain: KeychainKind::External,
                 is_spent: false,
@@ -763,17 +824,18 @@ mod test {
 
     fn generate_random_utxos(rng: &mut StdRng, utxos_number: usize) -> Vec<WeightedUtxo> {
         let mut res = Vec::new();
-        for _ in 0..utxos_number {
+        for i in 0..utxos_number {
             res.push(WeightedUtxo {
                 satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-                utxo: Utxo::Local(LocalUtxo {
-                    outpoint: OutPoint::from_str(
-                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
-                    )
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::from_str(&format!(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
+                        i
+                    ))
                     .unwrap(),
                     txout: TxOut {
-                        value: rng.gen_range(0..200000000),
-                        script_pubkey: Script::new(),
+                        value: Amount::from_sat(rng.gen_range(0..200000000)),
+                        script_pubkey: ScriptBuf::new(),
                     },
                     keychain: KeychainKind::External,
                     is_spent: false,
@@ -793,24 +855,26 @@ mod test {
     }
 
     fn generate_same_value_utxos(utxos_value: u64, utxos_number: usize) -> Vec<WeightedUtxo> {
-        let utxo = WeightedUtxo {
-            satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
-            utxo: Utxo::Local(LocalUtxo {
-                outpoint: OutPoint::from_str(
-                    "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:0",
-                )
-                .unwrap(),
-                txout: TxOut {
-                    value: utxos_value,
-                    script_pubkey: Script::new(),
-                },
-                keychain: KeychainKind::External,
-                is_spent: false,
-                derivation_index: 42,
-                confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
-            }),
-        };
-        vec![utxo; utxos_number]
+        (0..utxos_number)
+            .map(|i| WeightedUtxo {
+                satisfaction_weight: P2WPKH_SATISFACTION_SIZE,
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::from_str(&format!(
+                        "ebd9813ecebc57ff8f30797de7c205e3c7498ca950ea4341ee51a685ff2fa30a:{}",
+                        i
+                    ))
+                    .unwrap(),
+                    txout: TxOut {
+                        value: Amount::from_sat(utxos_value),
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    keychain: KeychainKind::External,
+                    is_spent: false,
+                    derivation_index: 42,
+                    confirmation_time: ConfirmationTime::Unconfirmed { last_seen: 0 },
+                }),
+            })
+            .collect()
     }
 
     fn sum_random_utxos(mut rng: &mut StdRng, utxos: &mut Vec<WeightedUtxo>) -> u64 {
@@ -818,21 +882,21 @@ mod test {
         utxos.shuffle(&mut rng);
         utxos[..utxos_picked_len]
             .iter()
-            .map(|u| u.utxo.txout().value)
+            .map(|u| u.utxo.txout().value.to_sat())
             .sum()
     }
 
     #[test]
     fn test_largest_first_coin_selection_success() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -846,14 +910,14 @@ mod test {
     #[test]
     fn test_largest_first_coin_selection_use_all() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -867,14 +931,14 @@ mod test {
     #[test]
     fn test_largest_first_coin_selection_use_only_necessary() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = LargestFirstCoinSelection::default()
+        let result = LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -889,14 +953,14 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_largest_first_coin_selection_insufficient_funds() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 500_000 + FEE_AMOUNT;
 
-        LargestFirstCoinSelection::default()
+        LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -907,14 +971,14 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_largest_first_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
-        LargestFirstCoinSelection::default()
+        LargestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -924,14 +988,14 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_success() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 180_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -945,14 +1009,14 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_use_all() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 utxos,
                 vec![],
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -966,14 +1030,14 @@ mod test {
     #[test]
     fn test_oldest_first_coin_selection_use_only_necessary() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let result = OldestFirstCoinSelection::default()
+        let result = OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -988,14 +1052,14 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_oldest_first_coin_selection_insufficient_funds() {
         let utxos = get_oldest_first_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 600_000 + FEE_AMOUNT;
 
-        OldestFirstCoinSelection::default()
+        OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1007,14 +1071,18 @@ mod test {
     fn test_oldest_first_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_oldest_first_test_utxos();
 
-        let target_amount: u64 = utxos.iter().map(|wu| wu.utxo.txout().value).sum::<u64>() - 50;
-        let drain_script = Script::default();
+        let target_amount: u64 = utxos
+            .iter()
+            .map(|wu| wu.utxo.txout().value.to_sat())
+            .sum::<u64>()
+            - 50;
+        let drain_script = ScriptBuf::default();
 
-        OldestFirstCoinSelection::default()
+        OldestFirstCoinSelection
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -1027,7 +1095,7 @@ mod test {
         // select three outputs
         let utxos = generate_same_value_utxos(100_000, 20);
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let target_amount = 250_000 + FEE_AMOUNT;
 
@@ -1035,7 +1103,7 @@ mod test {
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1049,14 +1117,14 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_required_are_enough() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
 
         let result = BranchAndBoundCoinSelection::default()
             .coin_select(
                 utxos.clone(),
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1070,14 +1138,14 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_optional_are_enough() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 299756 + FEE_AMOUNT;
 
         let result = BranchAndBoundCoinSelection::default()
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1102,11 +1170,11 @@ mod test {
         ));
 
         // Defensive assertions, for sanity and in case someone changes the test utxos vector.
-        let amount: u64 = required.iter().map(|u| u.utxo.txout().value).sum();
+        let amount: u64 = required.iter().map(|u| u.utxo.txout().value.to_sat()).sum();
         assert_eq!(amount, 100_000);
-        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value).sum();
+        let amount: u64 = optional.iter().map(|u| u.utxo.txout().value.to_sat()).sum();
         assert!(amount > 150_000);
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let target_amount = 150_000 + FEE_AMOUNT;
 
@@ -1114,7 +1182,7 @@ mod test {
             .coin_select(
                 required,
                 optional,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1129,14 +1197,14 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_bnb_coin_selection_insufficient_funds() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 500_000 + FEE_AMOUNT;
 
         BranchAndBoundCoinSelection::default()
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1.0),
+                FeeRate::from_sat_per_vb_unchecked(1),
                 target_amount,
                 &drain_script,
             )
@@ -1147,14 +1215,14 @@ mod test {
     #[should_panic(expected = "InsufficientFunds")]
     fn test_bnb_coin_selection_insufficient_funds_high_fees() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 250_000 + FEE_AMOUNT;
 
         BranchAndBoundCoinSelection::default()
             .coin_select(
                 vec![],
                 utxos,
-                FeeRate::from_sat_per_vb(1000.0),
+                FeeRate::from_sat_per_vb_unchecked(1000),
                 target_amount,
                 &drain_script,
             )
@@ -1164,24 +1232,21 @@ mod test {
     #[test]
     fn test_bnb_coin_selection_check_fee_rate() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 99932; // first utxo's effective value
+        let feerate = FeeRate::BROADCAST_MIN;
 
         let result = BranchAndBoundCoinSelection::new(0)
-            .coin_select(
-                vec![],
-                utxos,
-                FeeRate::from_sat_per_vb(1.0),
-                target_amount,
-                &drain_script,
-            )
+            .coin_select(vec![], utxos, feerate, target_amount, &drain_script)
             .unwrap();
 
         assert_eq!(result.selected.len(), 1);
         assert_eq!(result.selected_amount(), 100_000);
-        let input_size = (TXIN_BASE_WEIGHT + P2WPKH_SATISFACTION_SIZE).vbytes();
+        let input_weight =
+            TxIn::default().segwit_weight().to_wu() + P2WPKH_SATISFACTION_SIZE as u64;
         // the final fee rate should be exactly the same as the fee rate given
-        assert!((1.0 - (result.fee_amount as f32 / input_size as f32)).abs() < f32::EPSILON);
+        let result_feerate = Amount::from_sat(result.fee_amount) / Weight::from_wu(input_weight);
+        assert_eq!(result_feerate, feerate);
     }
 
     #[test]
@@ -1192,12 +1257,12 @@ mod test {
         for _i in 0..200 {
             let mut optional_utxos = generate_random_utxos(&mut rng, 16);
             let target_amount = sum_random_utxos(&mut rng, &mut optional_utxos);
-            let drain_script = Script::default();
+            let drain_script = ScriptBuf::default();
             let result = BranchAndBoundCoinSelection::new(0)
                 .coin_select(
                     vec![],
                     optional_utxos,
-                    FeeRate::from_sat_per_vb(0.0),
+                    FeeRate::ZERO,
                     target_amount,
                     &drain_script,
                 )
@@ -1209,7 +1274,7 @@ mod test {
     #[test]
     #[should_panic(expected = "BnBNoExactMatch")]
     fn test_bnb_function_no_exact_match() {
-        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(10);
         let utxos: Vec<OutputGroup> = get_test_utxos()
             .into_iter()
             .map(|u| OutputGroup::new(u, fee_rate))
@@ -1218,9 +1283,9 @@ mod test {
         let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
         let target_amount = 20_000 + FEE_AMOUNT;
         BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1239,7 +1304,7 @@ mod test {
     #[test]
     #[should_panic(expected = "BnBTotalTriesExceeded")]
     fn test_bnb_function_tries_exceeded() {
-        let fee_rate = FeeRate::from_sat_per_vb(10.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(10);
         let utxos: Vec<OutputGroup> = generate_same_value_utxos(100_000, 100_000)
             .into_iter()
             .map(|u| OutputGroup::new(u, fee_rate))
@@ -1248,10 +1313,10 @@ mod test {
         let curr_available_value = utxos.iter().fold(0, |acc, x| acc + x.effective_value);
 
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
         let target_amount = 20_000 + FEE_AMOUNT;
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1270,9 +1335,9 @@ mod test {
     // The match won't be exact but still in the range
     #[test]
     fn test_bnb_function_almost_exact_match_with_fees() {
-        let fee_rate = FeeRate::from_sat_per_vb(1.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(1);
         let size_of_change = 31;
-        let cost_of_change = size_of_change as f32 * fee_rate.as_sat_per_vb();
+        let cost_of_change = (Weight::from_vb_unchecked(size_of_change) * fee_rate).to_sat();
 
         let utxos: Vec<_> = generate_same_value_utxos(50_000, 10)
             .into_iter()
@@ -1285,9 +1350,9 @@ mod test {
 
         // 2*(value of 1 utxo)  - 2*(1 utxo fees with 1.0sat/vbyte fee rate) -
         // cost_of_change + 5.
-        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change.ceil() as i64 + 5;
+        let target_amount = 2 * 50_000 - 2 * 67 - cost_of_change as i64 + 5;
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let result = BranchAndBoundCoinSelection::new(size_of_change)
             .bnb(
@@ -1310,7 +1375,7 @@ mod test {
     fn test_bnb_function_exact_match_more_utxos() {
         let seed = [0; 32];
         let mut rng: StdRng = SeedableRng::from_seed(seed);
-        let fee_rate = FeeRate::from_sat_per_vb(0.0);
+        let fee_rate = FeeRate::ZERO;
 
         for _ in 0..200 {
             let optional_utxos: Vec<_> = generate_random_utxos(&mut rng, 40)
@@ -1327,7 +1392,7 @@ mod test {
             let target_amount =
                 optional_utxos[3].effective_value + optional_utxos[23].effective_value;
 
-            let drain_script = Script::default();
+            let drain_script = ScriptBuf::default();
 
             let result = BranchAndBoundCoinSelection::new(0)
                 .bnb(
@@ -1336,7 +1401,7 @@ mod test {
                     curr_value,
                     curr_available_value,
                     target_amount,
-                    0.0,
+                    0,
                     &drain_script,
                     fee_rate,
                 )
@@ -1352,13 +1417,13 @@ mod test {
         let mut utxos = generate_random_utxos(&mut rng, 300);
         let target_amount = sum_random_utxos(&mut rng, &mut utxos) + FEE_AMOUNT;
 
-        let fee_rate = FeeRate::from_sat_per_vb(1.0);
+        let fee_rate = FeeRate::from_sat_per_vb_unchecked(1);
         let utxos: Vec<OutputGroup> = utxos
             .into_iter()
             .map(|u| OutputGroup::new(u, fee_rate))
             .collect();
 
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let result = BranchAndBoundCoinSelection::default().single_random_draw(
             vec![],
@@ -1376,12 +1441,12 @@ mod test {
     #[test]
     fn test_bnb_exclude_negative_effective_value() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             vec![],
             utxos,
-            FeeRate::from_sat_per_vb(10.0),
+            FeeRate::from_sat_per_vb_unchecked(10),
             500_000,
             &drain_script,
         );
@@ -1398,16 +1463,16 @@ mod test {
     #[test]
     fn test_bnb_include_negative_effective_value_when_required() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
-        let (required, optional) = utxos
-            .into_iter()
-            .partition(|u| matches!(u, WeightedUtxo { utxo, .. } if utxo.txout().value < 1000));
+        let (required, optional) = utxos.into_iter().partition(
+            |u| matches!(u, WeightedUtxo { utxo, .. } if utxo.txout().value.to_sat() < 1000),
+        );
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             required,
             optional,
-            FeeRate::from_sat_per_vb(10.0),
+            FeeRate::from_sat_per_vb_unchecked(10),
             500_000,
             &drain_script,
         );
@@ -1424,12 +1489,12 @@ mod test {
     #[test]
     fn test_bnb_sum_of_effective_value_negative() {
         let utxos = get_test_utxos();
-        let drain_script = Script::default();
+        let drain_script = ScriptBuf::default();
 
         let selection = BranchAndBoundCoinSelection::default().coin_select(
             utxos,
             vec![],
-            FeeRate::from_sat_per_vb(10_000.0),
+            FeeRate::from_sat_per_vb_unchecked(10_000),
             500_000,
             &drain_script,
         );
@@ -1442,4 +1507,95 @@ mod test {
             })
         );
     }
+
+    #[test]
+    fn test_filter_duplicates() {
+        fn utxo(txid: &str, value: u64) -> WeightedUtxo {
+            WeightedUtxo {
+                satisfaction_weight: 0,
+                utxo: Utxo::Local(LocalOutput {
+                    outpoint: OutPoint::new(bitcoin::hashes::Hash::hash(txid.as_bytes()), 0),
+                    txout: TxOut {
+                        value: Amount::from_sat(value),
+                        script_pubkey: ScriptBuf::new(),
+                    },
+                    keychain: KeychainKind::External,
+                    is_spent: false,
+                    derivation_index: 0,
+                    confirmation_time: ConfirmationTime::Confirmed {
+                        height: 12345,
+                        time: 12345,
+                    },
+                }),
+            }
+        }
+
+        fn to_utxo_vec(utxos: &[(&str, u64)]) -> Vec<WeightedUtxo> {
+            let mut v = utxos
+                .iter()
+                .map(|&(txid, value)| utxo(txid, value))
+                .collect::<Vec<_>>();
+            v.sort_by_key(|u| u.utxo.outpoint());
+            v
+        }
+
+        struct TestCase<'a> {
+            name: &'a str,
+            required: &'a [(&'a str, u64)],
+            optional: &'a [(&'a str, u64)],
+            exp_required: &'a [(&'a str, u64)],
+            exp_optional: &'a [(&'a str, u64)],
+        }
+
+        let test_cases = [
+            TestCase {
+                name: "no_duplicates",
+                required: &[("A", 1000), ("B", 2100)],
+                optional: &[("C", 1000)],
+                exp_required: &[("A", 1000), ("B", 2100)],
+                exp_optional: &[("C", 1000)],
+            },
+            TestCase {
+                name: "duplicate_required_utxos",
+                required: &[("A", 3000), ("B", 1200), ("C", 1234), ("A", 3000)],
+                optional: &[("D", 2100)],
+                exp_required: &[("A", 3000), ("B", 1200), ("C", 1234)],
+                exp_optional: &[("D", 2100)],
+            },
+            TestCase {
+                name: "duplicate_optional_utxos",
+                required: &[("A", 3000), ("B", 1200)],
+                optional: &[("C", 5000), ("D", 1300), ("C", 5000)],
+                exp_required: &[("A", 3000), ("B", 1200)],
+                exp_optional: &[("C", 5000), ("D", 1300)],
+            },
+            TestCase {
+                name: "duplicate_across_required_and_optional_utxos",
+                required: &[("A", 3000), ("B", 1200), ("C", 2100)],
+                optional: &[("A", 3000), ("D", 1200), ("E", 5000)],
+                exp_required: &[("A", 3000), ("B", 1200), ("C", 2100)],
+                exp_optional: &[("D", 1200), ("E", 5000)],
+            },
+        ];
+
+        for (i, t) in test_cases.into_iter().enumerate() {
+            println!("Case {}: {}", i, t.name);
+            let (required, optional) =
+                filter_duplicates(to_utxo_vec(t.required), to_utxo_vec(t.optional));
+            assert_eq!(
+                required,
+                to_utxo_vec(t.exp_required),
+                "[{}:{}] unexpected `required` result",
+                i,
+                t.name
+            );
+            assert_eq!(
+                optional,
+                to_utxo_vec(t.exp_optional),
+                "[{}:{}] unexpected `optional` result",
+                i,
+                t.name
+            );
+        }
+    }
 }

crates/wallet/src/wallet/error.rs

@@ -0,0 +1,260 @@
+// Bitcoin Dev Kit
+// Written in 2020 by Alekos Filini <alekos.filini@gmail.com>
+//
+// Copyright (c) 2020-2021 Bitcoin Dev Kit Developers
+//
+// This file is licensed under the Apache License, Version 2.0 <LICENSE-APACHE
+// or http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your option.
+// You may not use this file except in accordance with one or both of these
+// licenses.
+
+//! Errors that can be thrown by the [`Wallet`](crate::wallet::Wallet)
+
+use crate::descriptor::policy::PolicyError;
+use crate::descriptor::DescriptorError;
+use crate::wallet::coin_selection;
+use crate::{descriptor, KeychainKind};
+use alloc::string::String;
+use bitcoin::{absolute, psbt, Amount, OutPoint, Sequence, Txid};
+use core::fmt;
+
+/// Errors returned by miniscript when updating inconsistent PSBTs
+#[derive(Debug, Clone)]
+pub enum MiniscriptPsbtError {
+    /// Descriptor key conversion error
+    Conversion(miniscript::descriptor::ConversionError),
+    /// Return error type for PsbtExt::update_input_with_descriptor
+    UtxoUpdate(miniscript::psbt::UtxoUpdateError),
+    /// Return error type for PsbtExt::update_output_with_descriptor
+    OutputUpdate(miniscript::psbt::OutputUpdateError),
+}
+
+impl fmt::Display for MiniscriptPsbtError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Conversion(err) => write!(f, "Conversion error: {}", err),
+            Self::UtxoUpdate(err) => write!(f, "UTXO update error: {}", err),
+            Self::OutputUpdate(err) => write!(f, "Output update error: {}", err),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for MiniscriptPsbtError {}
+
+#[derive(Debug)]
+/// Error returned from [`TxBuilder::finish`]
+///
+/// [`TxBuilder::finish`]: crate::wallet::tx_builder::TxBuilder::finish
+pub enum CreateTxError {
+    /// There was a problem with the descriptors passed in
+    Descriptor(DescriptorError),
+    /// There was a problem while extracting and manipulating policies
+    Policy(PolicyError),
+    /// Spending policy is not compatible with this [`KeychainKind`]
+    SpendingPolicyRequired(KeychainKind),
+    /// Requested invalid transaction version '0'
+    Version0,
+    /// Requested transaction version `1`, but at least `2` is needed to use OP_CSV
+    Version1Csv,
+    /// Requested `LockTime` is less than is required to spend from this script
+    LockTime {
+        /// Requested `LockTime`
+        requested: absolute::LockTime,
+        /// Required `LockTime`
+        required: absolute::LockTime,
+    },
+    /// Cannot enable RBF with a `Sequence` >= 0xFFFFFFFE
+    RbfSequence,
+    /// Cannot enable RBF with `Sequence` given a required OP_CSV
+    RbfSequenceCsv {
+        /// Given RBF `Sequence`
+        rbf: Sequence,
+        /// Required OP_CSV `Sequence`
+        csv: Sequence,
+    },
+    /// When bumping a tx the absolute fee requested is lower than replaced tx absolute fee
+    FeeTooLow {
+        /// Required fee absolute value [`Amount`]
+        required: Amount,
+    },
+    /// When bumping a tx the fee rate requested is lower than required
+    FeeRateTooLow {
+        /// Required fee rate
+        required: bitcoin::FeeRate,
+    },
+    /// `manually_selected_only` option is selected but no utxo has been passed
+    NoUtxosSelected,
+    /// Output created is under the dust limit, 546 satoshis
+    OutputBelowDustLimit(usize),
+    /// There was an error with coin selection
+    CoinSelection(coin_selection::Error),
+    /// Cannot build a tx without recipients
+    NoRecipients,
+    /// Partially signed bitcoin transaction error
+    Psbt(psbt::Error),
+    /// In order to use the [`TxBuilder::add_global_xpubs`] option every extended
+    /// key in the descriptor must either be a master key itself (having depth = 0) or have an
+    /// explicit origin provided
+    ///
+    /// [`TxBuilder::add_global_xpubs`]: crate::wallet::tx_builder::TxBuilder::add_global_xpubs
+    MissingKeyOrigin(String),
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo,
+    /// Missing non_witness_utxo on foreign utxo for given `OutPoint`
+    MissingNonWitnessUtxo(OutPoint),
+    /// Miniscript PSBT error
+    MiniscriptPsbt(MiniscriptPsbtError),
+}
+
+impl fmt::Display for CreateTxError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Descriptor(e) => e.fmt(f),
+            Self::Policy(e) => e.fmt(f),
+            CreateTxError::SpendingPolicyRequired(keychain_kind) => {
+                write!(f, "Spending policy required: {:?}", keychain_kind)
+            }
+            CreateTxError::Version0 => {
+                write!(f, "Invalid version `0`")
+            }
+            CreateTxError::Version1Csv => {
+                write!(
+                    f,
+                    "TxBuilder requested version `1`, but at least `2` is needed to use OP_CSV"
+                )
+            }
+            CreateTxError::LockTime {
+                requested,
+                required,
+            } => {
+                write!(f, "TxBuilder requested timelock of `{:?}`, but at least `{:?}` is required to spend from this script", required, requested)
+            }
+            CreateTxError::RbfSequence => {
+                write!(f, "Cannot enable RBF with a nSequence >= 0xFFFFFFFE")
+            }
+            CreateTxError::RbfSequenceCsv { rbf, csv } => {
+                write!(
+                    f,
+                    "Cannot enable RBF with nSequence `{:?}` given a required OP_CSV of `{:?}`",
+                    rbf, csv
+                )
+            }
+            CreateTxError::FeeTooLow { required } => {
+                write!(f, "Fee to low: required {}", required.display_dynamic())
+            }
+            CreateTxError::FeeRateTooLow { required } => {
+                write!(
+                    f,
+                    // Note: alternate fmt as sat/vb (ceil) available in bitcoin-0.31
+                    //"Fee rate too low: required {required:#}"
+                    "Fee rate too low: required {} sat/vb",
+                    crate::floating_rate!(required)
+                )
+            }
+            CreateTxError::NoUtxosSelected => {
+                write!(f, "No UTXO selected")
+            }
+            CreateTxError::OutputBelowDustLimit(limit) => {
+                write!(f, "Output below the dust limit: {}", limit)
+            }
+            CreateTxError::CoinSelection(e) => e.fmt(f),
+            CreateTxError::NoRecipients => {
+                write!(f, "Cannot build tx without recipients")
+            }
+            CreateTxError::Psbt(e) => e.fmt(f),
+            CreateTxError::MissingKeyOrigin(err) => {
+                write!(f, "Missing key origin: {}", err)
+            }
+            CreateTxError::UnknownUtxo => {
+                write!(f, "UTXO not found in the internal database")
+            }
+            CreateTxError::MissingNonWitnessUtxo(outpoint) => {
+                write!(f, "Missing non_witness_utxo on foreign utxo {}", outpoint)
+            }
+            CreateTxError::MiniscriptPsbt(err) => {
+                write!(f, "Miniscript PSBT error: {}", err)
+            }
+        }
+    }
+}
+
+impl From<descriptor::error::Error> for CreateTxError {
+    fn from(err: descriptor::error::Error) -> Self {
+        CreateTxError::Descriptor(err)
+    }
+}
+
+impl From<PolicyError> for CreateTxError {
+    fn from(err: PolicyError) -> Self {
+        CreateTxError::Policy(err)
+    }
+}
+
+impl From<MiniscriptPsbtError> for CreateTxError {
+    fn from(err: MiniscriptPsbtError) -> Self {
+        CreateTxError::MiniscriptPsbt(err)
+    }
+}
+
+impl From<psbt::Error> for CreateTxError {
+    fn from(err: psbt::Error) -> Self {
+        CreateTxError::Psbt(err)
+    }
+}
+
+impl From<coin_selection::Error> for CreateTxError {
+    fn from(err: coin_selection::Error) -> Self {
+        CreateTxError::CoinSelection(err)
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for CreateTxError {}
+
+#[derive(Debug)]
+/// Error returned from [`Wallet::build_fee_bump`]
+///
+/// [`Wallet::build_fee_bump`]: super::Wallet::build_fee_bump
+pub enum BuildFeeBumpError {
+    /// Happens when trying to spend an UTXO that is not in the internal database
+    UnknownUtxo(OutPoint),
+    /// Thrown when a tx is not found in the internal database
+    TransactionNotFound(Txid),
+    /// Happens when trying to bump a transaction that is already confirmed
+    TransactionConfirmed(Txid),
+    /// Trying to replace a tx that has a sequence >= `0xFFFFFFFE`
+    IrreplaceableTransaction(Txid),
+    /// Node doesn't have data to estimate a fee rate
+    FeeRateUnavailable,
+}
+
+impl fmt::Display for BuildFeeBumpError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::UnknownUtxo(outpoint) => write!(
+                f,
+                "UTXO not found in the internal database with txid: {}, vout: {}",
+                outpoint.txid, outpoint.vout
+            ),
+            Self::TransactionNotFound(txid) => {
+                write!(
+                    f,
+                    "Transaction not found in the internal database with txid: {}",
+                    txid
+                )
+            }
+            Self::TransactionConfirmed(txid) => {
+                write!(f, "Transaction already confirmed with txid: {}", txid)
+            }
+            Self::IrreplaceableTransaction(txid) => {
+                write!(f, "Transaction can't be replaced with txid: {}", txid)
+            }
+            Self::FeeRateUnavailable => write!(f, "Fee rate unavailable"),
+        }
+    }
+}
+
+#[cfg(feature = "std")]
+impl std::error::Error for BuildFeeBumpError {}

crates/wallet/src/wallet/export.rs

@@ -20,8 +20,8 @@
 //! ```
 //! # use std::str::FromStr;
 //! # use bitcoin::*;
-//! # use bdk::wallet::export::*;
-//! # use bdk::*;
+//! # use bdk_wallet::wallet::export::*;
+//! # use bdk_wallet::*;
 //! let import = r#"{
 //!     "descriptor": "wpkh([c258d2e4\/84h\/1h\/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe\/0\/*)",
 //!     "blockheight":1782088,
@@ -29,9 +29,9 @@
 //! }"#;
 //!
 //! let import = FullyNodedExport::from_str(import)?;
-//! let wallet = Wallet::new_no_persist(
+//! let wallet = Wallet::new(
 //!     &import.descriptor(),
-//!     import.change_descriptor().as_ref(),
+//!     &import.change_descriptor().expect("change descriptor"),
 //!     Network::Testnet,
 //! )?;
 //! # Ok::<_, Box<dyn std::error::Error>>(())
@@ -40,11 +40,11 @@
 //! ### Export a `Wallet`
 //! ```
 //! # use bitcoin::*;
-//! # use bdk::wallet::export::*;
-//! # use bdk::*;
-//! let wallet = Wallet::new_no_persist(
+//! # use bdk_wallet::wallet::export::*;
+//! # use bdk_wallet::*;
+//! let wallet = Wallet::new(
 //!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/0/*)",
-//!     Some("wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)"),
+//!     "wpkh([c258d2e4/84h/1h/0h]tpubDD3ynpHgJQW8VvWRzQ5WFDCrs4jqVFGHB3vLC3r49XHJSqP8bHKdK4AriuUKLccK68zfzowx7YhmDN8SiSkgCDENUFx9qVw65YyqM78vyVe/1/*)",
 //!     Network::Testnet,
 //! )?;
 //! let export = FullyNodedExport::export_wallet(&wallet, "exported wallet", true).unwrap();
@@ -53,9 +53,9 @@
 //! # Ok::<_, Box<dyn std::error::Error>>(())
 //! ```
 
+use alloc::string::String;
+use core::fmt;
 use core::str::FromStr;
-
-use alloc::string::{String, ToString};
 use serde::{Deserialize, Serialize};
 
 use miniscript::descriptor::{ShInner, WshInner};
@@ -80,9 +80,9 @@ pub struct FullyNodedExport {
     pub label: String,
 }
 
-impl ToString for FullyNodedExport {
-    fn to_string(&self) -> String {
-        serde_json::to_string(self).unwrap()
+impl fmt::Display for FullyNodedExport {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", serde_json::to_string(self).unwrap())
     }
 }
 
@@ -110,8 +110,8 @@ impl FullyNodedExport {
     ///
     /// If the database is empty or `include_blockheight` is false, the `blockheight` field
     /// returned will be `0`.
-    pub fn export_wallet<D>(
-        wallet: &Wallet<D>,
+    pub fn export_wallet(
+        wallet: &Wallet,
         label: &str,
         include_blockheight: bool,
     ) -> Result<Self, &'static str> {
@@ -126,13 +126,12 @@ impl FullyNodedExport {
         Self::is_compatible_with_core(&descriptor)?;
 
         let blockheight = if include_blockheight {
-            wallet
-                .transactions()
-                .next()
-                .map_or(0, |canonical_tx| match canonical_tx.observed_as {
+            wallet.transactions().next().map_or(0, |canonical_tx| {
+                match canonical_tx.chain_position {
                     bdk_chain::ChainPosition::Confirmed(a) => a.confirmation_height,
                     bdk_chain::ChainPosition::Unconfirmed(_) => 0,
-                })
+                }
+            })
         } else {
             0
         };
@@ -143,19 +142,17 @@ impl FullyNodedExport {
             blockheight,
         };
 
-        let change_descriptor = match wallet.public_descriptor(KeychainKind::Internal).is_some() {
-            false => None,
-            true => {
-                let descriptor = wallet
-                    .get_descriptor_for_keychain(KeychainKind::Internal)
-                    .to_string_with_secret(
-                        &wallet
-                            .get_signers(KeychainKind::Internal)
-                            .as_key_map(wallet.secp_ctx()),
-                    );
-                Some(remove_checksum(descriptor))
-            }
+        let change_descriptor = {
+            let descriptor = wallet
+                .get_descriptor_for_keychain(KeychainKind::Internal)
+                .to_string_with_secret(
+                    &wallet
+                        .get_signers(KeychainKind::Internal)
+                        .as_key_map(wallet.secp_ctx()),
+                );
+            Some(remove_checksum(descriptor))
         };
+
         if export.change_descriptor() != change_descriptor {
             return Err("Incompatible change descriptor");
         }
@@ -167,7 +164,7 @@ impl FullyNodedExport {
         fn check_ms<Ctx: ScriptContext>(
             terminal: &Terminal<String, Ctx>,
         ) -> Result<(), &'static str> {
-            if let Terminal::Multi(_, _) = terminal {
+            if let Terminal::Multi(_) = terminal {
                 Ok(())
             } else {
                 Err("The descriptor contains operators not supported by Bitcoin Core")
@@ -190,6 +187,7 @@ impl FullyNodedExport {
                 WshInner::SortedMulti(_) => Ok(()),
                 WshInner::Ms(ms) => check_ms(&ms.node),
             },
+            Descriptor::Tr(_) => Ok(()),
             _ => Err("The descriptor is not compatible with Bitcoin Core"),
         }
     }
@@ -215,24 +213,21 @@ impl FullyNodedExport {
 mod test {
     use core::str::FromStr;
 
+    use crate::std::string::ToString;
     use bdk_chain::{BlockId, ConfirmationTime};
     use bitcoin::hashes::Hash;
-    use bitcoin::{BlockHash, Network, Transaction};
+    use bitcoin::{transaction, BlockHash, Network, Transaction};
 
     use super::*;
     use crate::wallet::Wallet;
 
-    fn get_test_wallet(
-        descriptor: &str,
-        change_descriptor: Option<&str>,
-        network: Network,
-    ) -> Wallet<()> {
-        let mut wallet = Wallet::new_no_persist(descriptor, change_descriptor, network).unwrap();
+    fn get_test_wallet(descriptor: &str, change_descriptor: &str, network: Network) -> Wallet {
+        let mut wallet = Wallet::new(descriptor, change_descriptor, network).unwrap();
         let transaction = Transaction {
             input: vec![],
             output: vec![],
-            version: 0,
-            lock_time: bitcoin::PackedLockTime::ZERO,
+            version: transaction::Version::non_standard(0),
+            lock_time: bitcoin::absolute::LockTime::ZERO,
         };
         wallet
             .insert_checkpoint(BlockId {
@@ -257,7 +252,7 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.descriptor(), descriptor);
@@ -269,13 +264,14 @@ mod test {
     #[test]
     #[should_panic(expected = "Incompatible change descriptor")]
     fn test_export_no_change() {
-        // This wallet explicitly doesn't have a change descriptor. It should be impossible to
+        // The wallet's change descriptor has no wildcard. It should be impossible to
         // export, because exporting this kind of external descriptor normally implies the
-        // existence of an internal descriptor
+        // existence of a compatible internal descriptor
 
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
+        let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/0)";
 
-        let wallet = get_test_wallet(descriptor, None, Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -288,7 +284,7 @@ mod test {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/50'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
     }
 
@@ -305,7 +301,7 @@ mod test {
                                        [c98b1535/48'/0'/0'/2']tpubDCDi5W4sP6zSnzJeowy8rQDVhBdRARaPhK1axABi8V1661wEPeanpEXj4ZLAUEoikVtoWcyK26TKKJSecSfeKxwHCcRrge9k1ybuiL71z4a/1/*\
                                  ))";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Testnet);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Testnet);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.descriptor(), descriptor);
@@ -314,12 +310,24 @@ mod test {
         assert_eq!(export.label, "Test Label");
     }
 
+    #[test]
+    fn test_export_tr() {
+        let descriptor = "tr([73c5da0a/86'/0'/0']tprv8fMn4hSKPRC1oaCPqxDb1JWtgkpeiQvZhsr8W2xuy3GEMkzoArcAWTfJxYb6Wj8XNNDWEjfYKK4wGQXh3ZUXhDF2NcnsALpWTeSwarJt7Vc/0/*)";
+        let change_descriptor = "tr([73c5da0a/86'/0'/0']tprv8fMn4hSKPRC1oaCPqxDb1JWtgkpeiQvZhsr8W2xuy3GEMkzoArcAWTfJxYb6Wj8XNNDWEjfYKK4wGQXh3ZUXhDF2NcnsALpWTeSwarJt7Vc/1/*)";
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Testnet);
+        let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
+        assert_eq!(export.descriptor(), descriptor);
+        assert_eq!(export.change_descriptor(), Some(change_descriptor.into()));
+        assert_eq!(export.blockheight, 5000);
+        assert_eq!(export.label, "Test Label");
+    }
+
     #[test]
     fn test_export_to_json() {
         let descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/0/*)";
         let change_descriptor = "wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44'/0'/0'/1/*)";
 
-        let wallet = get_test_wallet(descriptor, Some(change_descriptor), Network::Bitcoin);
+        let wallet = get_test_wallet(descriptor, change_descriptor, Network::Bitcoin);
         let export = FullyNodedExport::export_wallet(&wallet, "Test Label", true).unwrap();
 
         assert_eq!(export.to_string(), "{\"descriptor\":\"wpkh(xprv9s21ZrQH143K4CTb63EaMxja1YiTnSEWKMbn23uoEnAzxjdUJRQkazCAtzxGm4LSoTSVTptoV9RbchnKPW9HxKtZumdyxyikZFDLhogJ5Uj/44\'/0\'/0\'/0/*)\",\"blockheight\":5000,\"label\":\"Test Label\"}");

crates/wallet/src/wallet/hardwaresigner.rs

@@ -14,12 +14,12 @@
 //! This module contains HWISigner, an implementation of a [TransactionSigner] to be
 //! used with hardware wallets.
 //! ```no_run
-//! # use bdk::bitcoin::Network;
-//! # use bdk::signer::SignerOrdering;
-//! # use bdk::wallet::hardwaresigner::HWISigner;
-//! # use bdk::wallet::AddressIndex::New;
-//! # use bdk::{FeeRate, KeychainKind, SignOptions, Wallet};
-//! # use hwi::{types::HWIChain, HWIClient};
+//! # use bdk_wallet::bitcoin::Network;
+//! # use bdk_wallet::signer::SignerOrdering;
+//! # use bdk_wallet::wallet::hardwaresigner::HWISigner;
+//! # use bdk_wallet::wallet::AddressIndex::New;
+//! # use bdk_wallet::{KeychainKind, SignOptions, Wallet};
+//! # use hwi::HWIClient;
 //! # use std::sync::Arc;
 //! #
 //! # fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -28,9 +28,9 @@
 //!     panic!("No devices found!");
 //! }
 //! let first_device = devices.remove(0)?;
-//! let custom_signer = HWISigner::from_device(&first_device, HWIChain::Test)?;
+//! let custom_signer = HWISigner::from_device(&first_device, Network::Testnet.into())?;
 //!
-//! # let mut wallet = Wallet::new_no_persist(
+//! # let mut wallet = Wallet::new(
 //! #     "",
 //! #     None,
 //! #     Network::Testnet,
@@ -47,9 +47,9 @@
 //! # }
 //! ```
 
-use bitcoin::psbt::PartiallySignedTransaction;
+use bitcoin::bip32::Fingerprint;
 use bitcoin::secp256k1::{All, Secp256k1};
-use bitcoin::util::bip32::Fingerprint;
+use bitcoin::Psbt;
 
 use hwi::error::Error;
 use hwi::types::{HWIChain, HWIDevice};
@@ -87,7 +87,7 @@ impl SignerCommon for HWISigner {
 impl TransactionSigner for HWISigner {
     fn sign_transaction(
         &self,
-        psbt: &mut PartiallySignedTransaction,
+        psbt: &mut Psbt,
         _sign_options: &crate::SignOptions,
         _secp: &crate::wallet::utils::SecpCtx,
     ) -> Result<(), SignerError> {